2024-11-13 05:39:33,957 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 05:39:33,974 main DEBUG Took 0.013220 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 05:39:33,974 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 05:39:33,975 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 05:39:33,976 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 05:39:33,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:39:33,988 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 05:39:34,005 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:39:34,007 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:39:34,008 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:39:34,008 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:39:34,009 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:39:34,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:39:34,011 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:39:34,012 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:39:34,012 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:39:34,013 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:39:34,014 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:39:34,015 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:39:34,015 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:39:34,016 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:39:34,017 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:39:34,017 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:39:34,018 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:39:34,018 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:39:34,019 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:39:34,019 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:39:34,020 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:39:34,020 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:39:34,021 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:39:34,021 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:39:34,022 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:39:34,023 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 05:39:34,025 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:39:34,027 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 05:39:34,029 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 05:39:34,030 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 05:39:34,032 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 05:39:34,032 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 05:39:34,042 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 05:39:34,046 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 05:39:34,049 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 05:39:34,049 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 05:39:34,050 main DEBUG createAppenders(={Console}) 2024-11-13 05:39:34,051 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-13 05:39:34,051 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 05:39:34,052 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-13 05:39:34,053 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 05:39:34,053 main DEBUG OutputStream closed 2024-11-13 05:39:34,053 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 05:39:34,054 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 05:39:34,054 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-13 05:39:34,172 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 05:39:34,175 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 05:39:34,177 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 05:39:34,178 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 05:39:34,179 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 05:39:34,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 05:39:34,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 05:39:34,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 05:39:34,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 05:39:34,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 05:39:34,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 05:39:34,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 05:39:34,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 05:39:34,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 05:39:34,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 05:39:34,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 05:39:34,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 05:39:34,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 05:39:34,189 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 05:39:34,190 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-13 05:39:34,190 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 05:39:34,191 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-13T05:39:34,535 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab 2024-11-13 05:39:34,539 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 05:39:34,539 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T05:39:34,553 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-13T05:39:34,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T05:39:34,663 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/cluster_5e8179f6-de47-de9a-0430-c54b93078463, deleteOnExit=true 2024-11-13T05:39:34,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T05:39:34,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/test.cache.data in system properties and HBase conf 2024-11-13T05:39:34,669 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T05:39:34,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/hadoop.log.dir in system properties and HBase conf 2024-11-13T05:39:34,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T05:39:34,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T05:39:34,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T05:39:34,799 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T05:39:34,936 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:39:34,941 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:39:34,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:39:34,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T05:39:34,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:39:34,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T05:39:34,945 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T05:39:34,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:39:34,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:39:34,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T05:39:34,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/nfs.dump.dir in system properties and HBase conf 2024-11-13T05:39:34,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/java.io.tmpdir in system properties and HBase conf 2024-11-13T05:39:34,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:39:34,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T05:39:34,952 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T05:39:36,132 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T05:39:36,216 INFO [Time-limited test {}] log.Log(170): Logging initialized @3147ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T05:39:36,308 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:39:36,402 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:39:36,428 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:39:36,428 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:39:36,430 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:39:36,442 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:39:36,446 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:39:36,447 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:39:36,669 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/java.io.tmpdir/jetty-localhost-35873-hadoop-hdfs-3_4_1-tests_jar-_-any-15926518091139660109/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:39:36,678 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:35873} 2024-11-13T05:39:36,678 INFO [Time-limited test {}] server.Server(415): Started @3610ms 2024-11-13T05:39:37,275 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:39:37,283 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:39:37,284 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:39:37,284 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:39:37,284 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:39:37,286 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:39:37,286 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:39:37,383 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/java.io.tmpdir/jetty-localhost-41021-hadoop-hdfs-3_4_1-tests_jar-_-any-3853838053954182267/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:39:37,384 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:41021} 2024-11-13T05:39:37,384 INFO [Time-limited test {}] server.Server(415): Started @4315ms 2024-11-13T05:39:37,431 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:39:38,296 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/cluster_5e8179f6-de47-de9a-0430-c54b93078463/data/data2/current/BP-218274341-172.17.0.2-1731476375683/current, will proceed with Du for space computation calculation, 2024-11-13T05:39:38,296 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/cluster_5e8179f6-de47-de9a-0430-c54b93078463/data/data1/current/BP-218274341-172.17.0.2-1731476375683/current, will proceed with Du for space computation calculation, 2024-11-13T05:39:38,323 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:39:38,365 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x367151f3a449107d with lease ID 0xe8c086040a36c435: Processing first storage report for DS-9008fbc6-4564-4f6c-8018-f5d6aafd2726 from datanode DatanodeRegistration(127.0.0.1:41403, datanodeUuid=ba6fbef7-2527-4c14-8925-e4f52a867c14, infoPort=40351, infoSecurePort=0, ipcPort=41353, storageInfo=lv=-57;cid=testClusterID;nsid=1702988596;c=1731476375683) 2024-11-13T05:39:38,367 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x367151f3a449107d with lease ID 0xe8c086040a36c435: from storage DS-9008fbc6-4564-4f6c-8018-f5d6aafd2726 node DatanodeRegistration(127.0.0.1:41403, datanodeUuid=ba6fbef7-2527-4c14-8925-e4f52a867c14, infoPort=40351, infoSecurePort=0, ipcPort=41353, storageInfo=lv=-57;cid=testClusterID;nsid=1702988596;c=1731476375683), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-13T05:39:38,367 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x367151f3a449107d with lease ID 0xe8c086040a36c435: Processing first storage report for DS-07936b8e-6b27-4147-8d63-46e6e7f4c8a5 from datanode DatanodeRegistration(127.0.0.1:41403, datanodeUuid=ba6fbef7-2527-4c14-8925-e4f52a867c14, infoPort=40351, infoSecurePort=0, ipcPort=41353, storageInfo=lv=-57;cid=testClusterID;nsid=1702988596;c=1731476375683) 2024-11-13T05:39:38,368 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x367151f3a449107d with lease ID 0xe8c086040a36c435: from storage DS-07936b8e-6b27-4147-8d63-46e6e7f4c8a5 node DatanodeRegistration(127.0.0.1:41403, datanodeUuid=ba6fbef7-2527-4c14-8925-e4f52a867c14, infoPort=40351, infoSecurePort=0, ipcPort=41353, storageInfo=lv=-57;cid=testClusterID;nsid=1702988596;c=1731476375683), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T05:39:38,394 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab 2024-11-13T05:39:38,467 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/cluster_5e8179f6-de47-de9a-0430-c54b93078463/zookeeper_0, clientPort=61732, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/cluster_5e8179f6-de47-de9a-0430-c54b93078463/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/cluster_5e8179f6-de47-de9a-0430-c54b93078463/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T05:39:38,476 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61732 2024-11-13T05:39:38,487 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:38,490 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:38,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:39:39,108 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745 with version=8 2024-11-13T05:39:39,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/hbase-staging 2024-11-13T05:39:39,189 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T05:39:39,467 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:39:39,476 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:39:39,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:39:39,481 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:39:39,482 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:39:39,482 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:39:39,642 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T05:39:39,710 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T05:39:39,719 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T05:39:39,722 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:39:39,745 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 996 (auto-detected) 2024-11-13T05:39:39,746 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-13T05:39:39,764 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37913 2024-11-13T05:39:39,786 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37913 connecting to ZooKeeper ensemble=127.0.0.1:61732 2024-11-13T05:39:39,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:379130x0, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:39:39,892 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37913-0x10133fe66110000 connected 2024-11-13T05:39:39,992 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:39,994 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:40,004 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:39:40,008 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745, hbase.cluster.distributed=false 2024-11-13T05:39:40,034 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:39:40,043 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37913 2024-11-13T05:39:40,043 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37913 2024-11-13T05:39:40,044 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37913 2024-11-13T05:39:40,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37913 2024-11-13T05:39:40,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37913 2024-11-13T05:39:40,168 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:39:40,170 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:39:40,171 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:39:40,171 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:39:40,171 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:39:40,171 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:39:40,175 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:39:40,178 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:39:40,180 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45837 2024-11-13T05:39:40,182 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45837 connecting to ZooKeeper ensemble=127.0.0.1:61732 2024-11-13T05:39:40,184 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:40,189 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:40,219 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:458370x0, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:39:40,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:458370x0, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:39:40,224 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:39:40,224 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45837-0x10133fe66110001 connected 2024-11-13T05:39:40,237 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:39:40,240 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:39:40,245 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:39:40,246 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45837 2024-11-13T05:39:40,247 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45837 2024-11-13T05:39:40,247 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45837 2024-11-13T05:39:40,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45837 2024-11-13T05:39:40,253 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45837 2024-11-13T05:39:40,273 DEBUG [M:0;bc386b7bf2fe:37913 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bc386b7bf2fe:37913 2024-11-13T05:39:40,275 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bc386b7bf2fe,37913,1731476379290 2024-11-13T05:39:40,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:39:40,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:39:40,309 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bc386b7bf2fe,37913,1731476379290 2024-11-13T05:39:40,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:40,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:39:40,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:40,344 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:39:40,345 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bc386b7bf2fe,37913,1731476379290 from backup master directory 2024-11-13T05:39:40,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bc386b7bf2fe,37913,1731476379290 2024-11-13T05:39:40,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:39:40,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:39:40,357 WARN [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:39:40,357 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bc386b7bf2fe,37913,1731476379290 2024-11-13T05:39:40,359 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T05:39:40,360 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T05:39:40,412 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/hbase.id] with ID: 0872bdfa-3c79-4391-8d17-f5ae243c6af2 2024-11-13T05:39:40,412 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/.tmp/hbase.id 2024-11-13T05:39:40,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:39:40,828 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/.tmp/hbase.id]:[hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/hbase.id] 2024-11-13T05:39:40,883 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:39:40,889 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T05:39:40,907 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-13T05:39:40,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:40,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:40,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:39:41,398 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:39:41,400 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T05:39:41,414 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:41,417 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:39:41,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:39:41,870 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store 2024-11-13T05:39:41,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:39:42,294 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T05:39:42,298 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:39:42,299 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:39:42,299 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:39:42,299 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:39:42,301 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:39:42,301 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:39:42,301 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:39:42,303 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476382299Disabling compacts and flushes for region at 1731476382299Disabling writes for close at 1731476382301 (+2 ms)Writing region close event to WAL at 1731476382301Closed at 1731476382301 2024-11-13T05:39:42,306 WARN [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/.initializing 2024-11-13T05:39:42,306 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/WALs/bc386b7bf2fe,37913,1731476379290 2024-11-13T05:39:42,315 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:39:42,330 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C37913%2C1731476379290, suffix=, logDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/WALs/bc386b7bf2fe,37913,1731476379290, archiveDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/oldWALs, maxLogs=10 2024-11-13T05:39:42,356 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/WALs/bc386b7bf2fe,37913,1731476379290/bc386b7bf2fe%2C37913%2C1731476379290.1731476382335, exclude list is [], retry=0 2024-11-13T05:39:42,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41403,DS-9008fbc6-4564-4f6c-8018-f5d6aafd2726,DISK] 2024-11-13T05:39:42,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T05:39:42,414 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/WALs/bc386b7bf2fe,37913,1731476379290/bc386b7bf2fe%2C37913%2C1731476379290.1731476382335 2024-11-13T05:39:42,415 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40351:40351)] 2024-11-13T05:39:42,416 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:39:42,416 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:39:42,419 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:42,420 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:42,457 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:42,482 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T05:39:42,486 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:42,489 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:42,490 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:42,494 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T05:39:42,494 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:42,496 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:39:42,496 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:42,499 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T05:39:42,500 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:42,501 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:39:42,502 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:42,505 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T05:39:42,505 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:42,507 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:39:42,507 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:42,511 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:42,512 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:42,517 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:42,518 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:42,522 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:39:42,527 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:39:42,532 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:39:42,534 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70273376, jitterRate=0.047154903411865234}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:39:42,542 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731476382432Initializing all the Stores at 1731476382434 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476382435 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476382436 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476382436Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476382436Cleaning up temporary data from old regions at 1731476382518 (+82 ms)Region opened successfully at 1731476382542 (+24 ms) 2024-11-13T05:39:42,543 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T05:39:42,581 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f113b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:39:42,624 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T05:39:42,641 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T05:39:42,641 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T05:39:42,645 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:39:42,646 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T05:39:42,653 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-13T05:39:42,653 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T05:39:42,685 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T05:39:42,697 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T05:39:42,755 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T05:39:42,758 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T05:39:42,760 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T05:39:42,772 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T05:39:42,774 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T05:39:42,778 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T05:39:42,788 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T05:39:42,790 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T05:39:42,801 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T05:39:42,823 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T05:39:42,834 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T05:39:42,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:39:42,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:39:42,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:42,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:42,851 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bc386b7bf2fe,37913,1731476379290, sessionid=0x10133fe66110000, setting cluster-up flag (Was=false) 2024-11-13T05:39:42,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:42,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:42,918 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T05:39:42,921 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bc386b7bf2fe,37913,1731476379290 2024-11-13T05:39:43,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:43,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:43,056 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T05:39:43,061 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bc386b7bf2fe,37913,1731476379290 2024-11-13T05:39:43,071 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T05:39:43,141 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:39:43,150 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T05:39:43,155 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T05:39:43,161 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(746): ClusterId : 0872bdfa-3c79-4391-8d17-f5ae243c6af2 2024-11-13T05:39:43,160 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bc386b7bf2fe,37913,1731476379290 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T05:39:43,163 DEBUG [RS:0;bc386b7bf2fe:45837 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:39:43,191 DEBUG [RS:0;bc386b7bf2fe:45837 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:39:43,191 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:39:43,191 DEBUG [RS:0;bc386b7bf2fe:45837 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:39:43,192 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:39:43,192 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:39:43,192 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:39:43,192 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bc386b7bf2fe:0, corePoolSize=10, maxPoolSize=10 2024-11-13T05:39:43,193 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:43,193 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:39:43,193 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:43,197 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731476413197 2024-11-13T05:39:43,200 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T05:39:43,200 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:39:43,200 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T05:39:43,201 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T05:39:43,202 DEBUG [RS:0;bc386b7bf2fe:45837 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:39:43,203 DEBUG [RS:0;bc386b7bf2fe:45837 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b395f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:39:43,204 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T05:39:43,205 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T05:39:43,205 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T05:39:43,205 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T05:39:43,207 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:43,207 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T05:39:43,208 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:43,213 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T05:39:43,214 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T05:39:43,215 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T05:39:43,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:39:43,217 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T05:39:43,217 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T05:39:43,219 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731476383219,5,FailOnTimeoutGroup] 2024-11-13T05:39:43,220 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bc386b7bf2fe:45837 2024-11-13T05:39:43,220 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731476383219,5,FailOnTimeoutGroup] 2024-11-13T05:39:43,220 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:43,220 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T05:39:43,221 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:43,221 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:43,222 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:39:43,222 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:39:43,223 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:39:43,225 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,37913,1731476379290 with port=45837, startcode=1731476380128 2024-11-13T05:39:43,233 DEBUG [RS:0;bc386b7bf2fe:45837 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:39:43,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58859, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:39:43,264 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37913 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:43,266 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37913 {}] master.ServerManager(517): Registering regionserver=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:43,278 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745 2024-11-13T05:39:43,278 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42759 2024-11-13T05:39:43,278 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:39:43,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:39:43,310 DEBUG [RS:0;bc386b7bf2fe:45837 {}] zookeeper.ZKUtil(111): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:43,310 WARN [RS:0;bc386b7bf2fe:45837 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:39:43,311 INFO [RS:0;bc386b7bf2fe:45837 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:39:43,311 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/WALs/bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:43,313 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bc386b7bf2fe,45837,1731476380128] 2024-11-13T05:39:43,338 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:39:43,353 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:39:43,357 INFO [RS:0;bc386b7bf2fe:45837 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:39:43,358 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:43,358 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:39:43,363 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:39:43,364 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:43,365 DEBUG [RS:0;bc386b7bf2fe:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:43,365 DEBUG [RS:0;bc386b7bf2fe:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:43,365 DEBUG [RS:0;bc386b7bf2fe:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:43,365 DEBUG [RS:0;bc386b7bf2fe:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:43,365 DEBUG [RS:0;bc386b7bf2fe:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:43,365 DEBUG [RS:0;bc386b7bf2fe:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:39:43,365 DEBUG [RS:0;bc386b7bf2fe:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:43,366 DEBUG [RS:0;bc386b7bf2fe:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:43,366 DEBUG [RS:0;bc386b7bf2fe:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:43,366 DEBUG [RS:0;bc386b7bf2fe:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:43,366 DEBUG [RS:0;bc386b7bf2fe:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:43,366 DEBUG [RS:0;bc386b7bf2fe:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:39:43,366 DEBUG [RS:0;bc386b7bf2fe:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:39:43,366 DEBUG [RS:0;bc386b7bf2fe:45837 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:39:43,367 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:43,368 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:43,368 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:43,368 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:43,368 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:43,368 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,45837,1731476380128-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:39:43,386 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:39:43,387 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,45837,1731476380128-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:43,387 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:43,388 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.Replication(171): bc386b7bf2fe,45837,1731476380128 started 2024-11-13T05:39:43,404 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:43,404 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1482): Serving as bc386b7bf2fe,45837,1731476380128, RpcServer on bc386b7bf2fe/172.17.0.2:45837, sessionid=0x10133fe66110001 2024-11-13T05:39:43,405 DEBUG [RS:0;bc386b7bf2fe:45837 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:39:43,405 DEBUG [RS:0;bc386b7bf2fe:45837 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:43,405 DEBUG [RS:0;bc386b7bf2fe:45837 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,45837,1731476380128' 2024-11-13T05:39:43,405 DEBUG [RS:0;bc386b7bf2fe:45837 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:39:43,406 DEBUG [RS:0;bc386b7bf2fe:45837 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:39:43,407 DEBUG [RS:0;bc386b7bf2fe:45837 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:39:43,407 DEBUG [RS:0;bc386b7bf2fe:45837 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:39:43,407 DEBUG [RS:0;bc386b7bf2fe:45837 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:43,407 DEBUG [RS:0;bc386b7bf2fe:45837 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,45837,1731476380128' 2024-11-13T05:39:43,407 DEBUG [RS:0;bc386b7bf2fe:45837 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:39:43,408 DEBUG [RS:0;bc386b7bf2fe:45837 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:39:43,408 DEBUG [RS:0;bc386b7bf2fe:45837 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:39:43,409 INFO [RS:0;bc386b7bf2fe:45837 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:39:43,409 INFO [RS:0;bc386b7bf2fe:45837 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:39:43,515 INFO [RS:0;bc386b7bf2fe:45837 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:39:43,519 INFO [RS:0;bc386b7bf2fe:45837 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C45837%2C1731476380128, suffix=, logDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/WALs/bc386b7bf2fe,45837,1731476380128, archiveDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/oldWALs, maxLogs=32 2024-11-13T05:39:43,535 DEBUG [RS:0;bc386b7bf2fe:45837 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/WALs/bc386b7bf2fe,45837,1731476380128/bc386b7bf2fe%2C45837%2C1731476380128.1731476383523, exclude list is [], retry=0 2024-11-13T05:39:43,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41403,DS-9008fbc6-4564-4f6c-8018-f5d6aafd2726,DISK] 2024-11-13T05:39:43,542 INFO [RS:0;bc386b7bf2fe:45837 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/WALs/bc386b7bf2fe,45837,1731476380128/bc386b7bf2fe%2C45837%2C1731476380128.1731476383523 2024-11-13T05:39:43,543 DEBUG [RS:0;bc386b7bf2fe:45837 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40351:40351)] 2024-11-13T05:39:43,620 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T05:39:43,621 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745 2024-11-13T05:39:43,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741833_1009 (size=32) 2024-11-13T05:39:44,037 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:39:44,043 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:39:44,046 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:39:44,047 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:44,048 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:44,048 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:39:44,052 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:39:44,052 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:44,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:44,054 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:39:44,057 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:39:44,058 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:44,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:44,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:39:44,061 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:39:44,061 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:44,062 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:44,063 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:39:44,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740 2024-11-13T05:39:44,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740 2024-11-13T05:39:44,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:39:44,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:39:44,068 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:39:44,070 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:39:44,074 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:39:44,075 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63677609, jitterRate=-0.05112968385219574}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:39:44,077 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731476384038Initializing all the Stores at 1731476384042 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476384042Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476384043 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476384043Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476384043Cleaning up temporary data from old regions at 1731476384067 (+24 ms)Region opened successfully at 1731476384077 (+10 ms) 2024-11-13T05:39:44,077 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:39:44,077 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:39:44,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:39:44,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:39:44,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:39:44,079 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:39:44,079 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476384077Disabling compacts and flushes for region at 1731476384077Disabling writes for close at 1731476384078 (+1 ms)Writing region close event to WAL at 1731476384078Closed at 1731476384079 (+1 ms) 2024-11-13T05:39:44,082 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:39:44,082 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T05:39:44,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:39:44,094 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:39:44,098 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T05:39:44,256 DEBUG [bc386b7bf2fe:37913 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T05:39:44,270 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:44,276 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,45837,1731476380128, state=OPENING 2024-11-13T05:39:44,338 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T05:39:44,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:44,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:39:44,349 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:39:44,349 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:39:44,351 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:39:44,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:39:44,528 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:39:44,531 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36319, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:39:44,542 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:39:44,543 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:39:44,543 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T05:39:44,546 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C45837%2C1731476380128.meta, suffix=.meta, logDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/WALs/bc386b7bf2fe,45837,1731476380128, archiveDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/oldWALs, maxLogs=32 2024-11-13T05:39:44,560 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/WALs/bc386b7bf2fe,45837,1731476380128/bc386b7bf2fe%2C45837%2C1731476380128.meta.1731476384548.meta, exclude list is [], retry=0 2024-11-13T05:39:44,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41403,DS-9008fbc6-4564-4f6c-8018-f5d6aafd2726,DISK] 2024-11-13T05:39:44,567 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/WALs/bc386b7bf2fe,45837,1731476380128/bc386b7bf2fe%2C45837%2C1731476380128.meta.1731476384548.meta 2024-11-13T05:39:44,567 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40351:40351)] 2024-11-13T05:39:44,567 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:39:44,569 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:39:44,572 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:39:44,576 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:39:44,580 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:39:44,580 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:39:44,581 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:39:44,581 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:39:44,584 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:39:44,586 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:39:44,586 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:44,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:44,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:39:44,588 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:39:44,588 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:44,589 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:44,589 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:39:44,591 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:39:44,591 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:44,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:44,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:39:44,593 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:39:44,593 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:44,594 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:39:44,594 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:39:44,596 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740 2024-11-13T05:39:44,598 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740 2024-11-13T05:39:44,601 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:39:44,601 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:39:44,602 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:39:44,605 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:39:44,607 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74545864, jitterRate=0.11081993579864502}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:39:44,607 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:39:44,609 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731476384581Writing region info on filesystem at 1731476384582 (+1 ms)Initializing all the Stores at 1731476384584 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476384584Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476384584Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476384584Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476384584Cleaning up temporary data from old regions at 1731476384601 (+17 ms)Running coprocessor post-open hooks at 1731476384608 (+7 ms)Region opened successfully at 1731476384609 (+1 ms) 2024-11-13T05:39:44,616 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731476384520 2024-11-13T05:39:44,627 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:39:44,628 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:39:44,630 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:44,633 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,45837,1731476380128, state=OPEN 2024-11-13T05:39:44,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:39:44,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:39:44,691 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:39:44,691 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:39:44,692 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:44,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T05:39:44,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45837,1731476380128 in 337 msec 2024-11-13T05:39:44,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T05:39:44,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 612 msec 2024-11-13T05:39:44,707 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:39:44,707 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T05:39:44,727 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:39:44,728 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:39:44,748 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:39:44,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50847, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:39:44,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6700 sec 2024-11-13T05:39:44,770 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731476384770, completionTime=-1 2024-11-13T05:39:44,773 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T05:39:44,773 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:39:44,797 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-13T05:39:44,798 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731476444798 2024-11-13T05:39:44,798 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731476504798 2024-11-13T05:39:44,798 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-13T05:39:44,800 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,37913,1731476379290-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:44,800 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,37913,1731476379290-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:44,800 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,37913,1731476379290-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:44,802 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bc386b7bf2fe:37913, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:44,802 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:44,803 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:44,809 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T05:39:44,831 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.474sec 2024-11-13T05:39:44,832 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T05:39:44,834 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T05:39:44,835 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T05:39:44,835 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T05:39:44,835 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T05:39:44,836 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,37913,1731476379290-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:39:44,836 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,37913,1731476379290-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T05:39:44,844 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T05:39:44,845 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T05:39:44,845 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,37913,1731476379290-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:39:44,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f081a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:44,876 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T05:39:44,876 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T05:39:44,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:39:44,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:39:44,890 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:39:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:39:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:39:44,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@531238f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:44,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:39:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:39:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:44,900 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:39:44,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62702105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:44,903 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:39:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:39:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:39:44,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:39:44,954 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bc386b7bf2fe,37913,1731476379290 2024-11-13T05:39:44,965 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=338, ProcessCount=11, AvailableMemoryMB=8023 2024-11-13T05:39:44,974 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:39:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,37913,1731476379290 2024-11-13T05:39:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@680215db 2024-11-13T05:39:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:39:44,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:39:44,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:39:44,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:39:45,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T05:39:45,009 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:39:45,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-13T05:39:45,011 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:45,015 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:39:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:39:45,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741835_1011 (size=959) 2024-11-13T05:39:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:39:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:39:45,433 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745 2024-11-13T05:39:45,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741836_1012 (size=53) 2024-11-13T05:39:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:39:45,845 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:39:45,845 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6abd824d4b7a94f2d19ab3e39e94ccfe, disabling compactions & flushes 2024-11-13T05:39:45,845 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:45,845 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:45,846 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. after waiting 0 ms 2024-11-13T05:39:45,846 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:45,846 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:45,846 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: Waiting for close lock at 1731476385845Disabling compacts and flushes for region at 1731476385845Disabling writes for close at 1731476385846 (+1 ms)Writing region close event to WAL at 1731476385846Closed at 1731476385846 2024-11-13T05:39:45,852 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:39:45,858 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731476385852"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731476385852"}]},"ts":"1731476385852"} 2024-11-13T05:39:45,864 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:39:45,866 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:39:45,868 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476385866"}]},"ts":"1731476385866"} 2024-11-13T05:39:45,873 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T05:39:45,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6abd824d4b7a94f2d19ab3e39e94ccfe, ASSIGN}] 2024-11-13T05:39:45,877 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6abd824d4b7a94f2d19ab3e39e94ccfe, ASSIGN 2024-11-13T05:39:45,879 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6abd824d4b7a94f2d19ab3e39e94ccfe, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45837,1731476380128; forceNewPlan=false, retain=false 2024-11-13T05:39:46,031 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6abd824d4b7a94f2d19ab3e39e94ccfe, regionState=OPENING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:46,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6abd824d4b7a94f2d19ab3e39e94ccfe, ASSIGN because future has completed 2024-11-13T05:39:46,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:39:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:39:46,206 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:46,206 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:39:46,207 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:46,207 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:39:46,207 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:46,207 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:46,210 INFO [StoreOpener-6abd824d4b7a94f2d19ab3e39e94ccfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:46,213 INFO [StoreOpener-6abd824d4b7a94f2d19ab3e39e94ccfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6abd824d4b7a94f2d19ab3e39e94ccfe columnFamilyName A 2024-11-13T05:39:46,213 DEBUG [StoreOpener-6abd824d4b7a94f2d19ab3e39e94ccfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:46,214 INFO [StoreOpener-6abd824d4b7a94f2d19ab3e39e94ccfe-1 {}] regionserver.HStore(327): Store=6abd824d4b7a94f2d19ab3e39e94ccfe/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:39:46,214 INFO [StoreOpener-6abd824d4b7a94f2d19ab3e39e94ccfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:46,216 INFO [StoreOpener-6abd824d4b7a94f2d19ab3e39e94ccfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6abd824d4b7a94f2d19ab3e39e94ccfe columnFamilyName B 2024-11-13T05:39:46,216 DEBUG [StoreOpener-6abd824d4b7a94f2d19ab3e39e94ccfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:46,217 INFO [StoreOpener-6abd824d4b7a94f2d19ab3e39e94ccfe-1 {}] regionserver.HStore(327): Store=6abd824d4b7a94f2d19ab3e39e94ccfe/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:39:46,217 INFO [StoreOpener-6abd824d4b7a94f2d19ab3e39e94ccfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:46,220 INFO [StoreOpener-6abd824d4b7a94f2d19ab3e39e94ccfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6abd824d4b7a94f2d19ab3e39e94ccfe columnFamilyName C 2024-11-13T05:39:46,220 DEBUG [StoreOpener-6abd824d4b7a94f2d19ab3e39e94ccfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:39:46,221 INFO [StoreOpener-6abd824d4b7a94f2d19ab3e39e94ccfe-1 {}] regionserver.HStore(327): Store=6abd824d4b7a94f2d19ab3e39e94ccfe/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:39:46,221 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:46,222 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:46,223 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:46,225 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:46,225 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:46,226 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:39:46,228 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:46,231 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:39:46,232 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6abd824d4b7a94f2d19ab3e39e94ccfe; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62433596, jitterRate=-0.06966692209243774}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:39:46,232 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:46,233 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: Running coprocessor pre-open hook at 1731476386207Writing region info on filesystem at 1731476386208 (+1 ms)Initializing all the Stores at 1731476386209 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476386210 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476386210Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476386210Cleaning up temporary data from old regions at 1731476386225 (+15 ms)Running coprocessor post-open hooks at 1731476386232 (+7 ms)Region opened successfully at 1731476386233 (+1 ms) 2024-11-13T05:39:46,235 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., pid=6, masterSystemTime=1731476386195 2024-11-13T05:39:46,238 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:46,239 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:46,240 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6abd824d4b7a94f2d19ab3e39e94ccfe, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:46,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:39:46,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T05:39:46,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 in 203 msec 2024-11-13T05:39:46,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T05:39:46,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6abd824d4b7a94f2d19ab3e39e94ccfe, ASSIGN in 375 msec 2024-11-13T05:39:46,256 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:39:46,256 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476386256"}]},"ts":"1731476386256"} 2024-11-13T05:39:46,259 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T05:39:46,261 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:39:46,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2620 sec 2024-11-13T05:39:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:39:47,180 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:39:47,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bafc0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:39:47,194 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:39:47,195 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:39:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:39:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:39:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1198e3f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:39:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:39:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:47,198 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:39:47,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@207fbc54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,200 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:39:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:39:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:39:47,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:39:47,207 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:39:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,37913,1731476379290 2024-11-13T05:39:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@451b8c3 2024-11-13T05:39:47,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:39:47,212 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:39:47,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706bb323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,217 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:39:47,217 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:39:47,218 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:39:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:39:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:39:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c85fb36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:39:47,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:39:47,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:47,221 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:39:47,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bee8057, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,223 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:39:47,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:39:47,225 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:39:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:39:47,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:39:47,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e11900a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:39:47,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:39:47,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:47,228 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:39:47,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f8ac40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,230 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:39:47,231 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:39:47,233 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:39:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:39:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:39:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25175df8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:39:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:39:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:47,237 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:39:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e59e9dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:39:47,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:39:47,241 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:39:47,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:39:47,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:39:47,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab7eb50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:39:47,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:39:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:47,245 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:39:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b025c7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,247 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:39:47,248 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:39:47,249 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:39:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:39:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:39:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c2308e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:39:47,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:39:47,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:47,251 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:39:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee049f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b274eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,254 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:39:47,254 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:39:47,255 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:39:47,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:39:47,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:39:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e9d3a67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:39:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:39:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:47,258 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:39:47,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d27e04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:39:47,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:39:47,261 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:39:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:39:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:39:47,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef2cbdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:39:47,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:39:47,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:47,263 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:39:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1beffb12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:39:47,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:39:47,266 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:39:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:39:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:39:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dfeb4cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:39:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:39:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:47,269 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:39:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6034ec4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,271 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:39:47,271 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:39:47,272 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:39:47,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:39:47,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:39:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3334cd5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:39:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:39:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:39:47,274 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:39:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eaf4ad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:39:47,280 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:39:47,280 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:39:47,280 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:39:47,281 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:39:47,282 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:39:47,283 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:39:47,283 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:39:47,283 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:39:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:39:47,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:39:47,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:39:47,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:39:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:39:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:39:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:39:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:39:47,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:39:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:39:47,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:39:47,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:39:47,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:39:47,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:39:47,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:39:47,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:39:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:39:47,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:39:47,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:39:47,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:39:47,290 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:39:47,291 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:39:47,292 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:39:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:39:47,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:39:47,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:39:47,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:39:47,295 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:39:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-13T05:39:47,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:39:47,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:39:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:39:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:39:47,299 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:39:47,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:39:47,302 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:39:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:39:47,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:39:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:39:47,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:39:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:39:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:39:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:39:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:47,381 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:47,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:39:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:39:47,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:47,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:47,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:47,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:47,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:47,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/fca911b09800409b918997af7426f2ad is 50, key is test_row_0/A:col10/1731476387384/Put/seqid=0 2024-11-13T05:39:47,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:47,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741837_1013 (size=16681) 2024-11-13T05:39:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476447533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476447551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476447555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476447548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476447563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:39:47,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:47,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:47,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:47,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:47,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:47,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:47,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476447698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476447698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476447698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476447707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476447707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:47,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:47,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:47,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:47,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:47,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:47,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476447908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476447917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476447918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476447918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476447919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:39:47,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/fca911b09800409b918997af7426f2ad 2024-11-13T05:39:48,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:48,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:48,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:48,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:48,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/c97f6407a97a445098ac1fa9823ef712 is 50, key is test_row_0/B:col10/1731476387384/Put/seqid=0 2024-11-13T05:39:48,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741838_1014 (size=12001) 2024-11-13T05:39:48,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/c97f6407a97a445098ac1fa9823ef712 2024-11-13T05:39:48,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/5e4b14f0baf143509d6bef4cbf220292 is 50, key is test_row_0/C:col10/1731476387384/Put/seqid=0 2024-11-13T05:39:48,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741839_1015 (size=12001) 2024-11-13T05:39:48,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:48,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:48,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:48,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:48,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476448227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476448238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476448239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476448239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476448239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:48,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:48,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:48,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:48,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:39:48,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:48,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:48,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:48,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:48,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/5e4b14f0baf143509d6bef4cbf220292 2024-11-13T05:39:48,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/fca911b09800409b918997af7426f2ad as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/fca911b09800409b918997af7426f2ad 2024-11-13T05:39:48,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/fca911b09800409b918997af7426f2ad, entries=250, sequenceid=14, filesize=16.3 K 2024-11-13T05:39:48,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/c97f6407a97a445098ac1fa9823ef712 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/c97f6407a97a445098ac1fa9823ef712 2024-11-13T05:39:48,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/c97f6407a97a445098ac1fa9823ef712, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T05:39:48,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/5e4b14f0baf143509d6bef4cbf220292 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5e4b14f0baf143509d6bef4cbf220292 2024-11-13T05:39:48,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5e4b14f0baf143509d6bef4cbf220292, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T05:39:48,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:48,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:48,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:48,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:48,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 1254ms, sequenceid=14, compaction requested=false 2024-11-13T05:39:48,644 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T05:39:48,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:48,744 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:48,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T05:39:48,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/e3472adb5d1142519f0b8b4df68e07e5 is 50, key is test_row_0/A:col10/1731476387551/Put/seqid=0 2024-11-13T05:39:48,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741840_1016 (size=14341) 2024-11-13T05:39:48,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/e3472adb5d1142519f0b8b4df68e07e5 2024-11-13T05:39:48,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476448778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:48,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:48,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:48,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:48,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476448782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476448782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/097b42762fa841b88d54145422962970 is 50, key is test_row_0/B:col10/1731476387551/Put/seqid=0 2024-11-13T05:39:48,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476448796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476448789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741841_1017 (size=12001) 2024-11-13T05:39:48,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/097b42762fa841b88d54145422962970 2024-11-13T05:39:48,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/01f87f6d5ad14092b4cc353b09ed8920 is 50, key is test_row_0/C:col10/1731476387551/Put/seqid=0 2024-11-13T05:39:48,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741842_1018 (size=12001) 2024-11-13T05:39:48,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/01f87f6d5ad14092b4cc353b09ed8920 2024-11-13T05:39:48,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476448897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/e3472adb5d1142519f0b8b4df68e07e5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/e3472adb5d1142519f0b8b4df68e07e5 2024-11-13T05:39:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476448907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476448917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476448928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476448929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:48,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/e3472adb5d1142519f0b8b4df68e07e5, entries=200, sequenceid=39, filesize=14.0 K 2024-11-13T05:39:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:48,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/097b42762fa841b88d54145422962970 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/097b42762fa841b88d54145422962970 2024-11-13T05:39:48,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/097b42762fa841b88d54145422962970, entries=150, sequenceid=39, filesize=11.7 K 2024-11-13T05:39:48,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/01f87f6d5ad14092b4cc353b09ed8920 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/01f87f6d5ad14092b4cc353b09ed8920 2024-11-13T05:39:48,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:48,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:48,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/01f87f6d5ad14092b4cc353b09ed8920, entries=150, sequenceid=39, filesize=11.7 K 2024-11-13T05:39:48,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 231ms, sequenceid=39, compaction requested=false 2024-11-13T05:39:48,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:49,111 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:49,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T05:39:49,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/2523210a3045447da24ac8566b91c633 is 50, key is test_row_0/A:col10/1731476388770/Put/seqid=0 2024-11-13T05:39:49,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741843_1019 (size=14341) 2024-11-13T05:39:49,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/2523210a3045447da24ac8566b91c633 2024-11-13T05:39:49,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/952b4e5d19cb4a968af9e0a3600c712c is 50, key is test_row_0/B:col10/1731476388770/Put/seqid=0 2024-11-13T05:39:49,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741844_1020 (size=12001) 2024-11-13T05:39:49,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/952b4e5d19cb4a968af9e0a3600c712c 2024-11-13T05:39:49,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/e13970f9ba76452f9c0086296986910d is 50, key is test_row_0/C:col10/1731476388770/Put/seqid=0 2024-11-13T05:39:49,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476449194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476449197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476449228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476449229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476449231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741845_1021 (size=12001) 2024-11-13T05:39:49,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/e13970f9ba76452f9c0086296986910d 2024-11-13T05:39:49,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/2523210a3045447da24ac8566b91c633 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/2523210a3045447da24ac8566b91c633 2024-11-13T05:39:49,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:49,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:49,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/2523210a3045447da24ac8566b91c633, entries=200, sequenceid=51, filesize=14.0 K 2024-11-13T05:39:49,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/952b4e5d19cb4a968af9e0a3600c712c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/952b4e5d19cb4a968af9e0a3600c712c 2024-11-13T05:39:49,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/952b4e5d19cb4a968af9e0a3600c712c, entries=150, sequenceid=51, filesize=11.7 K 2024-11-13T05:39:49,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/e13970f9ba76452f9c0086296986910d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/e13970f9ba76452f9c0086296986910d 2024-11-13T05:39:49,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/e13970f9ba76452f9c0086296986910d, entries=150, sequenceid=51, filesize=11.7 K 2024-11-13T05:39:49,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 211ms, sequenceid=51, compaction requested=true 2024-11-13T05:39:49,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:49,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:39:49,328 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:49,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:49,331 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:49,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:39:49,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:49,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:39:49,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:39:49,334 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T05:39:49,335 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:49,337 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/B is initiating minor compaction (all files) 2024-11-13T05:39:49,338 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/B in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,338 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/c97f6407a97a445098ac1fa9823ef712, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/097b42762fa841b88d54145422962970, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/952b4e5d19cb4a968af9e0a3600c712c] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=35.2 K 2024-11-13T05:39:49,341 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting c97f6407a97a445098ac1fa9823ef712, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731476387372 2024-11-13T05:39:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:49,341 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:49,342 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 097b42762fa841b88d54145422962970, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731476387530 2024-11-13T05:39:49,343 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 952b4e5d19cb4a968af9e0a3600c712c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731476388770 2024-11-13T05:39:49,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-13T05:39:49,351 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:49,351 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/A is initiating minor compaction (all files) 2024-11-13T05:39:49,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/32af0583b0ce4b7aafdb1d142f319d89 is 50, key is test_row_0/A:col10/1731476389342/Put/seqid=0 2024-11-13T05:39:49,364 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/A in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,365 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/fca911b09800409b918997af7426f2ad, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/e3472adb5d1142519f0b8b4df68e07e5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/2523210a3045447da24ac8566b91c633] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=44.3 K 2024-11-13T05:39:49,368 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting fca911b09800409b918997af7426f2ad, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731476387358 2024-11-13T05:39:49,369 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3472adb5d1142519f0b8b4df68e07e5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731476387530 2024-11-13T05:39:49,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476449356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476449357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476449358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476449369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476449372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,375 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2523210a3045447da24ac8566b91c633, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731476388770 2024-11-13T05:39:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741846_1022 (size=16681) 2024-11-13T05:39:49,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/32af0583b0ce4b7aafdb1d142f319d89 2024-11-13T05:39:49,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:49,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:49,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,431 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#B#compaction#10 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:49,432 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#A#compaction#11 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:49,432 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/1448118c03704b66a8a695d5b5885bfa is 50, key is test_row_0/B:col10/1731476388770/Put/seqid=0 2024-11-13T05:39:49,435 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/ff8facb8052b41da8b7645aee30a4d68 is 50, key is test_row_0/A:col10/1731476388770/Put/seqid=0 2024-11-13T05:39:49,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/6c05da63314c4191a8a7e62d11d59caf is 50, key is test_row_0/B:col10/1731476389342/Put/seqid=0 2024-11-13T05:39:49,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741847_1023 (size=12104) 2024-11-13T05:39:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:39:49,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741848_1024 (size=12104) 2024-11-13T05:39:49,469 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/1448118c03704b66a8a695d5b5885bfa as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1448118c03704b66a8a695d5b5885bfa 2024-11-13T05:39:49,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476449477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,485 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/ff8facb8052b41da8b7645aee30a4d68 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ff8facb8052b41da8b7645aee30a4d68 2024-11-13T05:39:49,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476449478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741849_1025 (size=12001) 2024-11-13T05:39:49,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/6c05da63314c4191a8a7e62d11d59caf 2024-11-13T05:39:49,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476449487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476449488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476449488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,507 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/B of 6abd824d4b7a94f2d19ab3e39e94ccfe into 1448118c03704b66a8a695d5b5885bfa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:49,507 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:49,510 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/B, priority=13, startTime=1731476389330; duration=0sec 2024-11-13T05:39:49,511 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:39:49,511 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:B 2024-11-13T05:39:49,511 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:49,517 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/A of 6abd824d4b7a94f2d19ab3e39e94ccfe into ff8facb8052b41da8b7645aee30a4d68(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:49,517 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:49,517 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/A, priority=13, startTime=1731476389324; duration=0sec 2024-11-13T05:39:49,518 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:49,518 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:A 2024-11-13T05:39:49,519 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:49,519 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/C is initiating minor compaction (all files) 2024-11-13T05:39:49,519 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/C in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,520 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5e4b14f0baf143509d6bef4cbf220292, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/01f87f6d5ad14092b4cc353b09ed8920, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/e13970f9ba76452f9c0086296986910d] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=35.2 K 2024-11-13T05:39:49,521 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e4b14f0baf143509d6bef4cbf220292, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731476387372 2024-11-13T05:39:49,524 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 01f87f6d5ad14092b4cc353b09ed8920, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731476387530 2024-11-13T05:39:49,525 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting e13970f9ba76452f9c0086296986910d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731476388770 2024-11-13T05:39:49,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/d3722b269be14cc28f54864fc3d50a2f is 50, key is test_row_0/C:col10/1731476389342/Put/seqid=0 2024-11-13T05:39:49,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741850_1026 (size=12001) 2024-11-13T05:39:49,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/d3722b269be14cc28f54864fc3d50a2f 2024-11-13T05:39:49,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:49,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:49,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,596 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#C#compaction#14 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:49,598 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/9b19a3df54cc40b6a766a45d717d7efb is 50, key is test_row_0/C:col10/1731476388770/Put/seqid=0 2024-11-13T05:39:49,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/32af0583b0ce4b7aafdb1d142f319d89 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/32af0583b0ce4b7aafdb1d142f319d89 2024-11-13T05:39:49,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/32af0583b0ce4b7aafdb1d142f319d89, entries=250, sequenceid=79, filesize=16.3 K 2024-11-13T05:39:49,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/6c05da63314c4191a8a7e62d11d59caf as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/6c05da63314c4191a8a7e62d11d59caf 2024-11-13T05:39:49,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/6c05da63314c4191a8a7e62d11d59caf, entries=150, sequenceid=79, filesize=11.7 K 2024-11-13T05:39:49,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/d3722b269be14cc28f54864fc3d50a2f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/d3722b269be14cc28f54864fc3d50a2f 2024-11-13T05:39:49,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741851_1027 (size=12104) 2024-11-13T05:39:49,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/d3722b269be14cc28f54864fc3d50a2f, entries=150, sequenceid=79, filesize=11.7 K 2024-11-13T05:39:49,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 322ms, sequenceid=79, compaction requested=false 2024-11-13T05:39:49,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:49,674 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/9b19a3df54cc40b6a766a45d717d7efb as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/9b19a3df54cc40b6a766a45d717d7efb 2024-11-13T05:39:49,701 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/C of 6abd824d4b7a94f2d19ab3e39e94ccfe into 9b19a3df54cc40b6a766a45d717d7efb(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:49,701 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:49,701 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/C, priority=13, startTime=1731476389332; duration=0sec 2024-11-13T05:39:49,702 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:49,702 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:C 2024-11-13T05:39:49,717 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:49,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T05:39:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:49,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/366e161d49224680b69f3d93f38de9fb is 50, key is test_row_0/A:col10/1731476389717/Put/seqid=0 2024-11-13T05:39:49,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:49,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:49,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741852_1028 (size=16681) 2024-11-13T05:39:49,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476449801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476449808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476449810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476449811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476449815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:49,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:49,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:49,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476449927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476449927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476449928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476449928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476449936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:49,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:50,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:50,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,094 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:39:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476450147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476450148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476450148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476450149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476450157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,176 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T05:39:50,176 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T05:39:50,179 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:39:50,179 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T05:39:50,179 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T05:39:50,179 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T05:39:50,181 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T05:39:50,181 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T05:39:50,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/366e161d49224680b69f3d93f38de9fb 2024-11-13T05:39:50,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:50,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:50,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/4a824387ff4742abaddb757aa5aeef51 is 50, key is test_row_0/B:col10/1731476389717/Put/seqid=0 2024-11-13T05:39:50,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741853_1029 (size=12001) 2024-11-13T05:39:50,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/4a824387ff4742abaddb757aa5aeef51 2024-11-13T05:39:50,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/b4d4568570794c47b9a2b2540b017200 is 50, key is test_row_0/C:col10/1731476389717/Put/seqid=0 2024-11-13T05:39:50,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741854_1030 (size=12001) 2024-11-13T05:39:50,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/b4d4568570794c47b9a2b2540b017200 2024-11-13T05:39:50,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:50,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:50,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/366e161d49224680b69f3d93f38de9fb as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/366e161d49224680b69f3d93f38de9fb 2024-11-13T05:39:50,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/366e161d49224680b69f3d93f38de9fb, entries=250, sequenceid=97, filesize=16.3 K 2024-11-13T05:39:50,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/4a824387ff4742abaddb757aa5aeef51 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/4a824387ff4742abaddb757aa5aeef51 2024-11-13T05:39:50,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/4a824387ff4742abaddb757aa5aeef51, entries=150, sequenceid=97, filesize=11.7 K 2024-11-13T05:39:50,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/b4d4568570794c47b9a2b2540b017200 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/b4d4568570794c47b9a2b2540b017200 2024-11-13T05:39:50,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/b4d4568570794c47b9a2b2540b017200, entries=150, sequenceid=97, filesize=11.7 K 2024-11-13T05:39:50,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 734ms, sequenceid=97, compaction requested=true 2024-11-13T05:39:50,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:50,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:39:50,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:50,451 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:50,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:39:50,451 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:50,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:50,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:39:50,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:39:50,471 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45466 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:50,471 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/A is initiating minor compaction (all files) 2024-11-13T05:39:50,471 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:50,471 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/B is initiating minor compaction (all files) 2024-11-13T05:39:50,471 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/A in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,471 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/B in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,471 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ff8facb8052b41da8b7645aee30a4d68, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/32af0583b0ce4b7aafdb1d142f319d89, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/366e161d49224680b69f3d93f38de9fb] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=44.4 K 2024-11-13T05:39:50,472 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1448118c03704b66a8a695d5b5885bfa, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/6c05da63314c4191a8a7e62d11d59caf, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/4a824387ff4742abaddb757aa5aeef51] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=35.3 K 2024-11-13T05:39:50,473 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff8facb8052b41da8b7645aee30a4d68, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731476388770 2024-11-13T05:39:50,473 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:50,473 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 1448118c03704b66a8a695d5b5885bfa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731476388770 2024-11-13T05:39:50,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T05:39:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:50,474 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32af0583b0ce4b7aafdb1d142f319d89, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731476389204 2024-11-13T05:39:50,475 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c05da63314c4191a8a7e62d11d59caf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731476389204 2024-11-13T05:39:50,475 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 366e161d49224680b69f3d93f38de9fb, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731476389711 2024-11-13T05:39:50,478 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a824387ff4742abaddb757aa5aeef51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731476389712 2024-11-13T05:39:50,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/0d98e4e53e1044c48bb6fa5d8bbb83b9 is 50, key is test_row_0/A:col10/1731476389810/Put/seqid=0 2024-11-13T05:39:50,504 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#A#compaction#19 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:50,505 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/8f1636ff445449b38c52d452a437fd14 is 50, key is test_row_0/A:col10/1731476389717/Put/seqid=0 2024-11-13T05:39:50,518 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#B#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:50,519 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/4ed3bd516c744fd59a9be270924bb512 is 50, key is test_row_0/B:col10/1731476389717/Put/seqid=0 2024-11-13T05:39:50,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476450505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476450507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476450510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476450514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476450521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:50,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:50,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741855_1031 (size=14341) 2024-11-13T05:39:50,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741856_1032 (size=12207) 2024-11-13T05:39:50,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741857_1033 (size=12207) 2024-11-13T05:39:50,624 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/4ed3bd516c744fd59a9be270924bb512 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/4ed3bd516c744fd59a9be270924bb512 2024-11-13T05:39:50,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476450626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476450626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476450627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476450627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476450636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,642 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/B of 6abd824d4b7a94f2d19ab3e39e94ccfe into 4ed3bd516c744fd59a9be270924bb512(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:50,642 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:50,642 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/B, priority=13, startTime=1731476390451; duration=0sec 2024-11-13T05:39:50,643 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:39:50,643 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:B 2024-11-13T05:39:50,643 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:50,646 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:50,647 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/C is initiating minor compaction (all files) 2024-11-13T05:39:50,647 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/C in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,647 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/9b19a3df54cc40b6a766a45d717d7efb, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/d3722b269be14cc28f54864fc3d50a2f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/b4d4568570794c47b9a2b2540b017200] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=35.3 K 2024-11-13T05:39:50,649 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b19a3df54cc40b6a766a45d717d7efb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731476388770 2024-11-13T05:39:50,650 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting d3722b269be14cc28f54864fc3d50a2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731476389204 2024-11-13T05:39:50,651 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting b4d4568570794c47b9a2b2540b017200, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731476389712 2024-11-13T05:39:50,670 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:50,671 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/137fbaad12c04d9eab7bb0885673ac2d is 50, key is test_row_0/C:col10/1731476389717/Put/seqid=0 2024-11-13T05:39:50,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:50,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:50,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741858_1034 (size=12207) 2024-11-13T05:39:50,725 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/137fbaad12c04d9eab7bb0885673ac2d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/137fbaad12c04d9eab7bb0885673ac2d 2024-11-13T05:39:50,753 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/C of 6abd824d4b7a94f2d19ab3e39e94ccfe into 137fbaad12c04d9eab7bb0885673ac2d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:50,753 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:50,753 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/C, priority=13, startTime=1731476390452; duration=0sec 2024-11-13T05:39:50,754 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:50,754 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:C 2024-11-13T05:39:50,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476450837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476450837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476450837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476450837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:50,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:50,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:50,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476450847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:50,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/0d98e4e53e1044c48bb6fa5d8bbb83b9 2024-11-13T05:39:50,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/f22c4c5c5cae4b87a536765ed56a54f3 is 50, key is test_row_0/B:col10/1731476389810/Put/seqid=0 2024-11-13T05:39:50,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:50,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:50,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:50,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:51,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:51,002 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/8f1636ff445449b38c52d452a437fd14 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8f1636ff445449b38c52d452a437fd14 2024-11-13T05:39:51,016 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/A of 6abd824d4b7a94f2d19ab3e39e94ccfe into 8f1636ff445449b38c52d452a437fd14(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:51,016 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:51,016 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/A, priority=13, startTime=1731476390451; duration=0sec 2024-11-13T05:39:51,017 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:51,017 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:A 2024-11-13T05:39:51,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741859_1035 (size=12001) 2024-11-13T05:39:51,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/f22c4c5c5cae4b87a536765ed56a54f3 2024-11-13T05:39:51,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/ddba5ab719964ab1ba9de7c1dab9e1bb is 50, key is test_row_0/C:col10/1731476389810/Put/seqid=0 2024-11-13T05:39:51,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741860_1036 (size=12001) 2024-11-13T05:39:51,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/ddba5ab719964ab1ba9de7c1dab9e1bb 2024-11-13T05:39:51,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/0d98e4e53e1044c48bb6fa5d8bbb83b9 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0d98e4e53e1044c48bb6fa5d8bbb83b9 2024-11-13T05:39:51,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:51,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:51,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:51,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476451147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0d98e4e53e1044c48bb6fa5d8bbb83b9, entries=200, sequenceid=119, filesize=14.0 K 2024-11-13T05:39:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476451148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476451148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/f22c4c5c5cae4b87a536765ed56a54f3 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f22c4c5c5cae4b87a536765ed56a54f3 2024-11-13T05:39:51,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476451156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476451147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f22c4c5c5cae4b87a536765ed56a54f3, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T05:39:51,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/ddba5ab719964ab1ba9de7c1dab9e1bb as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/ddba5ab719964ab1ba9de7c1dab9e1bb 2024-11-13T05:39:51,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/ddba5ab719964ab1ba9de7c1dab9e1bb, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T05:39:51,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 733ms, sequenceid=119, compaction requested=false 2024-11-13T05:39:51,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:51,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:39:51,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:51,316 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T05:39:51,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/73d0ca667cd7496aa5d95676726513d2 is 50, key is test_row_0/A:col10/1731476390509/Put/seqid=0 2024-11-13T05:39:51,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741861_1037 (size=12151) 2024-11-13T05:39:51,363 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/73d0ca667cd7496aa5d95676726513d2 2024-11-13T05:39:51,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/f22979d52a414451b2804b0a468cdf47 is 50, key is test_row_0/B:col10/1731476390509/Put/seqid=0 2024-11-13T05:39:51,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741862_1038 (size=12151) 2024-11-13T05:39:51,463 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/f22979d52a414451b2804b0a468cdf47 2024-11-13T05:39:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:39:51,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/59477aff38c941818a138be7b675df89 is 50, key is test_row_0/C:col10/1731476390509/Put/seqid=0 2024-11-13T05:39:51,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741863_1039 (size=12151) 2024-11-13T05:39:51,554 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/59477aff38c941818a138be7b675df89 2024-11-13T05:39:51,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/73d0ca667cd7496aa5d95676726513d2 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/73d0ca667cd7496aa5d95676726513d2 2024-11-13T05:39:51,579 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/73d0ca667cd7496aa5d95676726513d2, entries=150, sequenceid=136, filesize=11.9 K 2024-11-13T05:39:51,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/f22979d52a414451b2804b0a468cdf47 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f22979d52a414451b2804b0a468cdf47 2024-11-13T05:39:51,593 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f22979d52a414451b2804b0a468cdf47, entries=150, sequenceid=136, filesize=11.9 K 2024-11-13T05:39:51,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/59477aff38c941818a138be7b675df89 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/59477aff38c941818a138be7b675df89 2024-11-13T05:39:51,608 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/59477aff38c941818a138be7b675df89, entries=150, sequenceid=136, filesize=11.9 K 2024-11-13T05:39:51,610 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 294ms, sequenceid=136, compaction requested=true 2024-11-13T05:39:51,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:51,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:51,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-13T05:39:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-13T05:39:51,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T05:39:51,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.3080 sec 2024-11-13T05:39:51,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 4.3360 sec 2024-11-13T05:39:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:51,709 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:51,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:39:51,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/8a7351059e4741138fcdf6496231a3df is 50, key is test_row_0/A:col10/1731476391703/Put/seqid=0 2024-11-13T05:39:51,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741864_1040 (size=14537) 2024-11-13T05:39:51,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/8a7351059e4741138fcdf6496231a3df 2024-11-13T05:39:51,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/abfdcc7ea82c4d3c8f2ac144967821dc is 50, key is test_row_0/B:col10/1731476391703/Put/seqid=0 2024-11-13T05:39:51,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476451771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476451780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476451790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476451791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476451794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741865_1041 (size=9757) 2024-11-13T05:39:51,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/abfdcc7ea82c4d3c8f2ac144967821dc 2024-11-13T05:39:51,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/d5742a3a530346d7b98d7b7caf08f076 is 50, key is test_row_0/C:col10/1731476391703/Put/seqid=0 2024-11-13T05:39:51,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741866_1042 (size=9757) 2024-11-13T05:39:51,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/d5742a3a530346d7b98d7b7caf08f076 2024-11-13T05:39:51,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476451906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476451917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476451918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476451918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/8a7351059e4741138fcdf6496231a3df as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8a7351059e4741138fcdf6496231a3df 2024-11-13T05:39:51,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:51,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476451918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:51,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:51,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8a7351059e4741138fcdf6496231a3df, entries=200, sequenceid=147, filesize=14.2 K 2024-11-13T05:39:51,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/abfdcc7ea82c4d3c8f2ac144967821dc as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/abfdcc7ea82c4d3c8f2ac144967821dc 2024-11-13T05:39:51,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/abfdcc7ea82c4d3c8f2ac144967821dc, entries=100, sequenceid=147, filesize=9.5 K 2024-11-13T05:39:51,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/d5742a3a530346d7b98d7b7caf08f076 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/d5742a3a530346d7b98d7b7caf08f076 2024-11-13T05:39:51,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/d5742a3a530346d7b98d7b7caf08f076, entries=100, sequenceid=147, filesize=9.5 K 2024-11-13T05:39:51,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 265ms, sequenceid=147, compaction requested=true 2024-11-13T05:39:51,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:51,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:39:51,975 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:39:51,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:51,975 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:39:51,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:39:51,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:51,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:39:51,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:39:51,977 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:39:51,978 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/B is initiating minor compaction (all files) 2024-11-13T05:39:51,978 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/B in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:51,978 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/4ed3bd516c744fd59a9be270924bb512, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f22c4c5c5cae4b87a536765ed56a54f3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f22979d52a414451b2804b0a468cdf47, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/abfdcc7ea82c4d3c8f2ac144967821dc] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=45.0 K 2024-11-13T05:39:51,978 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53236 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:39:51,979 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/A is initiating minor compaction (all files) 2024-11-13T05:39:51,979 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/A in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:51,979 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8f1636ff445449b38c52d452a437fd14, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0d98e4e53e1044c48bb6fa5d8bbb83b9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/73d0ca667cd7496aa5d95676726513d2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8a7351059e4741138fcdf6496231a3df] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=52.0 K 2024-11-13T05:39:51,979 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ed3bd516c744fd59a9be270924bb512, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731476389712 2024-11-13T05:39:51,980 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f1636ff445449b38c52d452a437fd14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731476389712 2024-11-13T05:39:51,980 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting f22c4c5c5cae4b87a536765ed56a54f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731476389759 2024-11-13T05:39:51,981 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting f22979d52a414451b2804b0a468cdf47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731476390503 2024-11-13T05:39:51,982 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting abfdcc7ea82c4d3c8f2ac144967821dc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731476391703 2024-11-13T05:39:51,986 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d98e4e53e1044c48bb6fa5d8bbb83b9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731476389759 2024-11-13T05:39:51,990 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73d0ca667cd7496aa5d95676726513d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731476390503 2024-11-13T05:39:51,992 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a7351059e4741138fcdf6496231a3df, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731476391689 2024-11-13T05:39:52,033 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#A#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:52,034 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/163896c8328140d780d45295b7eb17e9 is 50, key is test_row_0/A:col10/1731476391703/Put/seqid=0 2024-11-13T05:39:52,047 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#B#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:52,048 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/5a84fb8e8cef4e04b44e56ceaebb9b6b is 50, key is test_row_0/B:col10/1731476391703/Put/seqid=0 2024-11-13T05:39:52,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741867_1043 (size=12493) 2024-11-13T05:39:52,083 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/163896c8328140d780d45295b7eb17e9 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/163896c8328140d780d45295b7eb17e9 2024-11-13T05:39:52,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741868_1044 (size=12493) 2024-11-13T05:39:52,110 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/A of 6abd824d4b7a94f2d19ab3e39e94ccfe into 163896c8328140d780d45295b7eb17e9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:52,110 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:52,110 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/A, priority=12, startTime=1731476391975; duration=0sec 2024-11-13T05:39:52,110 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:39:52,110 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:A 2024-11-13T05:39:52,111 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:39:52,115 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/5a84fb8e8cef4e04b44e56ceaebb9b6b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/5a84fb8e8cef4e04b44e56ceaebb9b6b 2024-11-13T05:39:52,122 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:39:52,122 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/C is initiating minor compaction (all files) 2024-11-13T05:39:52,122 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/C in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:52,123 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/137fbaad12c04d9eab7bb0885673ac2d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/ddba5ab719964ab1ba9de7c1dab9e1bb, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/59477aff38c941818a138be7b675df89, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/d5742a3a530346d7b98d7b7caf08f076] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=45.0 K 2024-11-13T05:39:52,125 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 137fbaad12c04d9eab7bb0885673ac2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731476389712 2024-11-13T05:39:52,126 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddba5ab719964ab1ba9de7c1dab9e1bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731476389759 2024-11-13T05:39:52,127 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59477aff38c941818a138be7b675df89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731476390503 2024-11-13T05:39:52,127 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5742a3a530346d7b98d7b7caf08f076, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731476391703 2024-11-13T05:39:52,128 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/B of 6abd824d4b7a94f2d19ab3e39e94ccfe into 5a84fb8e8cef4e04b44e56ceaebb9b6b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:52,128 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:52,128 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/B, priority=12, startTime=1731476391975; duration=0sec 2024-11-13T05:39:52,129 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:52,129 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:B 2024-11-13T05:39:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:52,132 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:52,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T05:39:52,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/6cf05076cfa54190a8fe43ad192bde7e is 50, key is test_row_0/A:col10/1731476392129/Put/seqid=0 2024-11-13T05:39:52,162 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:52,163 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/5a370675bbbf44af821482dbfb97cafe is 50, key is test_row_0/C:col10/1731476391703/Put/seqid=0 2024-11-13T05:39:52,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476452158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476452165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476452169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476452173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476452196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741869_1045 (size=14541) 2024-11-13T05:39:52,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/6cf05076cfa54190a8fe43ad192bde7e 2024-11-13T05:39:52,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741870_1046 (size=12493) 2024-11-13T05:39:52,233 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/5a370675bbbf44af821482dbfb97cafe as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5a370675bbbf44af821482dbfb97cafe 2024-11-13T05:39:52,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/eb0276bb8b814f91a020b89f42819e4e is 50, key is test_row_0/B:col10/1731476392129/Put/seqid=0 2024-11-13T05:39:52,247 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/C of 6abd824d4b7a94f2d19ab3e39e94ccfe into 5a370675bbbf44af821482dbfb97cafe(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:52,247 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:52,248 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/C, priority=12, startTime=1731476391977; duration=0sec 2024-11-13T05:39:52,248 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:52,248 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:C 2024-11-13T05:39:52,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741871_1047 (size=12151) 2024-11-13T05:39:52,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476452288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476452288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476452288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476452298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476452306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476452497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476452497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476452506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476452517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476452527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/eb0276bb8b814f91a020b89f42819e4e 2024-11-13T05:39:52,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/5ae150d869504c53a9bc8406491af2f1 is 50, key is test_row_0/C:col10/1731476392129/Put/seqid=0 2024-11-13T05:39:52,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741872_1048 (size=12151) 2024-11-13T05:39:52,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476452807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476452816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476452817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476452826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:52,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476452837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:52,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:52,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/5ae150d869504c53a9bc8406491af2f1 2024-11-13T05:39:53,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/6cf05076cfa54190a8fe43ad192bde7e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/6cf05076cfa54190a8fe43ad192bde7e 2024-11-13T05:39:53,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/6cf05076cfa54190a8fe43ad192bde7e, entries=200, sequenceid=175, filesize=14.2 K 2024-11-13T05:39:53,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/eb0276bb8b814f91a020b89f42819e4e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/eb0276bb8b814f91a020b89f42819e4e 2024-11-13T05:39:53,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/eb0276bb8b814f91a020b89f42819e4e, entries=150, sequenceid=175, filesize=11.9 K 2024-11-13T05:39:53,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/5ae150d869504c53a9bc8406491af2f1 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5ae150d869504c53a9bc8406491af2f1 2024-11-13T05:39:53,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5ae150d869504c53a9bc8406491af2f1, entries=150, sequenceid=175, filesize=11.9 K 2024-11-13T05:39:53,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 1114ms, sequenceid=175, compaction requested=false 2024-11-13T05:39:53,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:53,332 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:53,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T05:39:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:53,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/14f9631a69ab4c03b7bdc91407218cdb is 50, key is test_row_0/A:col10/1731476392147/Put/seqid=0 2024-11-13T05:39:53,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741873_1049 (size=12151) 2024-11-13T05:39:53,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/14f9631a69ab4c03b7bdc91407218cdb 2024-11-13T05:39:53,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/ca76f3b8c5ad4a52ad78a3e665592639 is 50, key is test_row_0/B:col10/1731476392147/Put/seqid=0 2024-11-13T05:39:53,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476453393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476453401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476453403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476453405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476453405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741874_1050 (size=12151) 2024-11-13T05:39:53,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/ca76f3b8c5ad4a52ad78a3e665592639 2024-11-13T05:39:53,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/4e16b06384844b1293e412addb9f335e is 50, key is test_row_0/C:col10/1731476392147/Put/seqid=0 2024-11-13T05:39:53,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476453517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476453523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741875_1051 (size=12151) 2024-11-13T05:39:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476453523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476453523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476453526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476453728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476453737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476453737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476453737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476453737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:53,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:53,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/4e16b06384844b1293e412addb9f335e 2024-11-13T05:39:53,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/14f9631a69ab4c03b7bdc91407218cdb as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/14f9631a69ab4c03b7bdc91407218cdb 2024-11-13T05:39:53,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/14f9631a69ab4c03b7bdc91407218cdb, entries=150, sequenceid=188, filesize=11.9 K 2024-11-13T05:39:53,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/ca76f3b8c5ad4a52ad78a3e665592639 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/ca76f3b8c5ad4a52ad78a3e665592639 2024-11-13T05:39:53,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/ca76f3b8c5ad4a52ad78a3e665592639, entries=150, sequenceid=188, filesize=11.9 K 2024-11-13T05:39:53,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/4e16b06384844b1293e412addb9f335e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/4e16b06384844b1293e412addb9f335e 2024-11-13T05:39:53,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/4e16b06384844b1293e412addb9f335e, entries=150, sequenceid=188, filesize=11.9 K 2024-11-13T05:39:53,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 647ms, sequenceid=188, compaction requested=true 2024-11-13T05:39:53,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:53,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:39:53,980 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:53,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:53,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:39:53,980 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:53,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:53,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:39:53,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:39:53,982 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:53,982 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/B is initiating minor compaction (all files) 2024-11-13T05:39:53,982 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/B in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:53,982 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/5a84fb8e8cef4e04b44e56ceaebb9b6b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/eb0276bb8b814f91a020b89f42819e4e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/ca76f3b8c5ad4a52ad78a3e665592639] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=35.9 K 2024-11-13T05:39:53,983 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:53,983 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/A is initiating minor compaction (all files) 2024-11-13T05:39:53,983 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a84fb8e8cef4e04b44e56ceaebb9b6b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731476390503 2024-11-13T05:39:53,983 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/A in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:53,983 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/163896c8328140d780d45295b7eb17e9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/6cf05076cfa54190a8fe43ad192bde7e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/14f9631a69ab4c03b7bdc91407218cdb] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=38.3 K 2024-11-13T05:39:53,984 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting eb0276bb8b814f91a020b89f42819e4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731476391751 2024-11-13T05:39:53,984 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 163896c8328140d780d45295b7eb17e9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731476390503 2024-11-13T05:39:53,985 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cf05076cfa54190a8fe43ad192bde7e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731476391751 2024-11-13T05:39:53,985 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting ca76f3b8c5ad4a52ad78a3e665592639, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731476392147 2024-11-13T05:39:53,985 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14f9631a69ab4c03b7bdc91407218cdb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731476392147 2024-11-13T05:39:54,021 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:54,022 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/a09b4bb1f63a4e898ffae7803f05d433 is 50, key is test_row_0/A:col10/1731476392147/Put/seqid=0 2024-11-13T05:39:54,033 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:54,034 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/641cdeb4bf614571822695b27cbe513b is 50, key is test_row_0/B:col10/1731476392147/Put/seqid=0 2024-11-13T05:39:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:54,041 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:54,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T05:39:54,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741876_1052 (size=12595) 2024-11-13T05:39:54,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476454062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476454064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476454066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476454067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476454067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/0ce75d7010db4f0192676d211c2f46fe is 50, key is test_row_0/A:col10/1731476393400/Put/seqid=0 2024-11-13T05:39:54,100 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/a09b4bb1f63a4e898ffae7803f05d433 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/a09b4bb1f63a4e898ffae7803f05d433 2024-11-13T05:39:54,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741877_1053 (size=12595) 2024-11-13T05:39:54,124 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/A of 6abd824d4b7a94f2d19ab3e39e94ccfe into a09b4bb1f63a4e898ffae7803f05d433(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:54,124 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:54,125 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/A, priority=13, startTime=1731476393979; duration=0sec 2024-11-13T05:39:54,125 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:39:54,125 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:A 2024-11-13T05:39:54,125 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:54,127 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:54,127 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/C is initiating minor compaction (all files) 2024-11-13T05:39:54,127 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/C in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:54,128 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5a370675bbbf44af821482dbfb97cafe, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5ae150d869504c53a9bc8406491af2f1, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/4e16b06384844b1293e412addb9f335e] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=35.9 K 2024-11-13T05:39:54,129 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a370675bbbf44af821482dbfb97cafe, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731476390503 2024-11-13T05:39:54,130 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ae150d869504c53a9bc8406491af2f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731476391751 2024-11-13T05:39:54,131 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e16b06384844b1293e412addb9f335e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731476392147 2024-11-13T05:39:54,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741878_1054 (size=14541) 2024-11-13T05:39:54,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/0ce75d7010db4f0192676d211c2f46fe 2024-11-13T05:39:54,165 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:54,166 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/98c44411a3744de0bffb83ba28fcb4b3 is 50, key is test_row_0/C:col10/1731476392147/Put/seqid=0 2024-11-13T05:39:54,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476454178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/674230cd601042d5a4661765f47235e7 is 50, key is test_row_0/B:col10/1731476393400/Put/seqid=0 2024-11-13T05:39:54,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476454187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476454188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476454188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476454189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741879_1055 (size=12595) 2024-11-13T05:39:54,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741880_1056 (size=12151) 2024-11-13T05:39:54,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/674230cd601042d5a4661765f47235e7 2024-11-13T05:39:54,256 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/98c44411a3744de0bffb83ba28fcb4b3 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/98c44411a3744de0bffb83ba28fcb4b3 2024-11-13T05:39:54,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/5ed3142deb404c5d9430d7e06df7e815 is 50, key is test_row_0/C:col10/1731476393400/Put/seqid=0 2024-11-13T05:39:54,277 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/C of 6abd824d4b7a94f2d19ab3e39e94ccfe into 98c44411a3744de0bffb83ba28fcb4b3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:54,281 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:54,281 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/C, priority=13, startTime=1731476393980; duration=0sec 2024-11-13T05:39:54,281 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:54,281 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:C 2024-11-13T05:39:54,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741881_1057 (size=12151) 2024-11-13T05:39:54,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/5ed3142deb404c5d9430d7e06df7e815 2024-11-13T05:39:54,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/0ce75d7010db4f0192676d211c2f46fe as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0ce75d7010db4f0192676d211c2f46fe 2024-11-13T05:39:54,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0ce75d7010db4f0192676d211c2f46fe, entries=200, sequenceid=213, filesize=14.2 K 2024-11-13T05:39:54,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/674230cd601042d5a4661765f47235e7 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/674230cd601042d5a4661765f47235e7 2024-11-13T05:39:54,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/674230cd601042d5a4661765f47235e7, entries=150, sequenceid=213, filesize=11.9 K 2024-11-13T05:39:54,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/5ed3142deb404c5d9430d7e06df7e815 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5ed3142deb404c5d9430d7e06df7e815 2024-11-13T05:39:54,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5ed3142deb404c5d9430d7e06df7e815, entries=150, sequenceid=213, filesize=11.9 K 2024-11-13T05:39:54,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 320ms, sequenceid=213, compaction requested=false 2024-11-13T05:39:54,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:54,448 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:54,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T05:39:54,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/7f9f5cdb145044d4ae9275b0749eb6c0 is 50, key is test_row_0/A:col10/1731476394402/Put/seqid=0 2024-11-13T05:39:54,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741882_1058 (size=12151) 2024-11-13T05:39:54,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/7f9f5cdb145044d4ae9275b0749eb6c0 2024-11-13T05:39:54,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476454509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,530 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/641cdeb4bf614571822695b27cbe513b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/641cdeb4bf614571822695b27cbe513b 2024-11-13T05:39:54,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476454515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/1f1c05158cae4416860497ce4d1ecf7b is 50, key is test_row_0/B:col10/1731476394402/Put/seqid=0 2024-11-13T05:39:54,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476454504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476454526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476454528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,546 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/B of 6abd824d4b7a94f2d19ab3e39e94ccfe into 641cdeb4bf614571822695b27cbe513b(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:54,547 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:54,547 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/B, priority=13, startTime=1731476393980; duration=0sec 2024-11-13T05:39:54,547 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:54,547 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:B 2024-11-13T05:39:54,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741883_1059 (size=12151) 2024-11-13T05:39:54,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/1f1c05158cae4416860497ce4d1ecf7b 2024-11-13T05:39:54,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/faeddd80e7db4b5d9279c56f79e8b967 is 50, key is test_row_0/C:col10/1731476394402/Put/seqid=0 2024-11-13T05:39:54,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741884_1060 (size=12151) 2024-11-13T05:39:54,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/faeddd80e7db4b5d9279c56f79e8b967 2024-11-13T05:39:54,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476454648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476454648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/7f9f5cdb145044d4ae9275b0749eb6c0 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/7f9f5cdb145044d4ae9275b0749eb6c0 2024-11-13T05:39:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476454648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476454648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476454648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/7f9f5cdb145044d4ae9275b0749eb6c0, entries=150, sequenceid=229, filesize=11.9 K 2024-11-13T05:39:54,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/1f1c05158cae4416860497ce4d1ecf7b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1f1c05158cae4416860497ce4d1ecf7b 2024-11-13T05:39:54,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1f1c05158cae4416860497ce4d1ecf7b, entries=150, sequenceid=229, filesize=11.9 K 2024-11-13T05:39:54,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/faeddd80e7db4b5d9279c56f79e8b967 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/faeddd80e7db4b5d9279c56f79e8b967 2024-11-13T05:39:54,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/faeddd80e7db4b5d9279c56f79e8b967, entries=150, sequenceid=229, filesize=11.9 K 2024-11-13T05:39:54,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 243ms, sequenceid=229, compaction requested=true 2024-11-13T05:39:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:39:54,692 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:39:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:54,692 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:39:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:39:54,695 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:54,695 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/A is initiating minor compaction (all files) 2024-11-13T05:39:54,696 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/A in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:54,696 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/a09b4bb1f63a4e898ffae7803f05d433, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0ce75d7010db4f0192676d211c2f46fe, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/7f9f5cdb145044d4ae9275b0749eb6c0] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=38.4 K 2024-11-13T05:39:54,696 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:54,696 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/B is initiating minor compaction (all files) 2024-11-13T05:39:54,696 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/B in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:54,697 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/641cdeb4bf614571822695b27cbe513b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/674230cd601042d5a4661765f47235e7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1f1c05158cae4416860497ce4d1ecf7b] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=36.0 K 2024-11-13T05:39:54,698 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 641cdeb4bf614571822695b27cbe513b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731476392147 2024-11-13T05:39:54,698 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting a09b4bb1f63a4e898ffae7803f05d433, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731476392147 2024-11-13T05:39:54,698 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ce75d7010db4f0192676d211c2f46fe, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731476393390 2024-11-13T05:39:54,699 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f9f5cdb145044d4ae9275b0749eb6c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731476394063 2024-11-13T05:39:54,699 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 674230cd601042d5a4661765f47235e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731476393400 2024-11-13T05:39:54,700 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f1c05158cae4416860497ce4d1ecf7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731476394063 2024-11-13T05:39:54,714 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:54,715 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/1c12e7ae8671427d849746ea5729a4a2 is 50, key is test_row_0/B:col10/1731476394402/Put/seqid=0 2024-11-13T05:39:54,719 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:54,719 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/ec92832c4e9c4404ae614285a7f31d5f is 50, key is test_row_0/A:col10/1731476394402/Put/seqid=0 2024-11-13T05:39:54,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741885_1061 (size=12697) 2024-11-13T05:39:54,750 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/1c12e7ae8671427d849746ea5729a4a2 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1c12e7ae8671427d849746ea5729a4a2 2024-11-13T05:39:54,760 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/B of 6abd824d4b7a94f2d19ab3e39e94ccfe into 1c12e7ae8671427d849746ea5729a4a2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:54,760 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:54,761 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/B, priority=13, startTime=1731476394692; duration=0sec 2024-11-13T05:39:54,761 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:39:54,761 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:B 2024-11-13T05:39:54,761 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:39:54,763 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:39:54,763 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/C is initiating minor compaction (all files) 2024-11-13T05:39:54,763 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/C in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:54,763 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/98c44411a3744de0bffb83ba28fcb4b3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5ed3142deb404c5d9430d7e06df7e815, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/faeddd80e7db4b5d9279c56f79e8b967] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=36.0 K 2024-11-13T05:39:54,765 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 98c44411a3744de0bffb83ba28fcb4b3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731476392147 2024-11-13T05:39:54,766 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ed3142deb404c5d9430d7e06df7e815, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731476393400 2024-11-13T05:39:54,768 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting faeddd80e7db4b5d9279c56f79e8b967, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731476394063 2024-11-13T05:39:54,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741886_1062 (size=12697) 2024-11-13T05:39:54,781 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/ec92832c4e9c4404ae614285a7f31d5f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ec92832c4e9c4404ae614285a7f31d5f 2024-11-13T05:39:54,800 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/A of 6abd824d4b7a94f2d19ab3e39e94ccfe into ec92832c4e9c4404ae614285a7f31d5f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:54,800 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:54,800 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/A, priority=13, startTime=1731476394692; duration=0sec 2024-11-13T05:39:54,801 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:54,801 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:A 2024-11-13T05:39:54,801 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#C#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:54,802 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/c3c84ebdae074927935dfd0f3c83ff3e is 50, key is test_row_0/C:col10/1731476394402/Put/seqid=0 2024-11-13T05:39:54,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741887_1063 (size=12697) 2024-11-13T05:39:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:54,860 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:54,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T05:39:54,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/5bd931cfa250485fb861377f24d9a918 is 50, key is test_row_0/A:col10/1731476394523/Put/seqid=0 2024-11-13T05:39:54,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476454881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476454887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476454888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476454890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476454892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:54,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741888_1064 (size=16981) 2024-11-13T05:39:55,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476454997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476454997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476454997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476455007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476455007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476455217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476455217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476455217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476455218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476455223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,239 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/c3c84ebdae074927935dfd0f3c83ff3e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/c3c84ebdae074927935dfd0f3c83ff3e 2024-11-13T05:39:55,255 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/C of 6abd824d4b7a94f2d19ab3e39e94ccfe into c3c84ebdae074927935dfd0f3c83ff3e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:55,255 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:55,256 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/C, priority=13, startTime=1731476394692; duration=0sec 2024-11-13T05:39:55,257 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:55,257 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:C 2024-11-13T05:39:55,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/5bd931cfa250485fb861377f24d9a918 2024-11-13T05:39:55,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/7985a8244c5145809993c562e36c4111 is 50, key is test_row_0/B:col10/1731476394523/Put/seqid=0 2024-11-13T05:39:55,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741889_1065 (size=12201) 2024-11-13T05:39:55,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/7985a8244c5145809993c562e36c4111 2024-11-13T05:39:55,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/945d45fc464d48d2ba913fe2f8c34cd8 is 50, key is test_row_0/C:col10/1731476394523/Put/seqid=0 2024-11-13T05:39:55,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741890_1066 (size=12201) 2024-11-13T05:39:55,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/945d45fc464d48d2ba913fe2f8c34cd8 2024-11-13T05:39:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:39:55,479 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:39:55,480 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:39:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-13T05:39:55,483 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:39:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:39:55,485 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:39:55,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:39:55,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/5bd931cfa250485fb861377f24d9a918 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5bd931cfa250485fb861377f24d9a918 2024-11-13T05:39:55,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5bd931cfa250485fb861377f24d9a918, entries=250, sequenceid=257, filesize=16.6 K 2024-11-13T05:39:55,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/7985a8244c5145809993c562e36c4111 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/7985a8244c5145809993c562e36c4111 2024-11-13T05:39:55,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/7985a8244c5145809993c562e36c4111, entries=150, sequenceid=257, filesize=11.9 K 2024-11-13T05:39:55,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/945d45fc464d48d2ba913fe2f8c34cd8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/945d45fc464d48d2ba913fe2f8c34cd8 2024-11-13T05:39:55,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476455526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476455527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476455527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476455527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476455563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:55,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/945d45fc464d48d2ba913fe2f8c34cd8, entries=150, sequenceid=257, filesize=11.9 K 2024-11-13T05:39:55,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 712ms, sequenceid=257, compaction requested=false 2024-11-13T05:39:55,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:39:55,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T05:39:55,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:55,640 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:39:55,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/5f893eee04fb424ca62d15b8a2038564 is 50, key is test_row_0/A:col10/1731476394889/Put/seqid=0 2024-11-13T05:39:55,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741891_1067 (size=12301) 2024-11-13T05:39:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:39:56,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:56,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476456093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,110 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/5f893eee04fb424ca62d15b8a2038564 2024-11-13T05:39:56,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476456100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476456102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476456102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:39:56,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476456102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/b11fbe021c544a8f8b4766b6dcfbfa38 is 50, key is test_row_0/B:col10/1731476394889/Put/seqid=0 2024-11-13T05:39:56,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741892_1068 (size=12301) 2024-11-13T05:39:56,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476456207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476456216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476456216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476456218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476456226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476456417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476456428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476456428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476456436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476456436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,547 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/b11fbe021c544a8f8b4766b6dcfbfa38 2024-11-13T05:39:56,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/38ad84b2e2784f52abbe497de242c6ff is 50, key is test_row_0/C:col10/1731476394889/Put/seqid=0 2024-11-13T05:39:56,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741893_1069 (size=12301) 2024-11-13T05:39:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:39:56,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476456726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476456739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476456737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476456746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476456746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:56,971 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/38ad84b2e2784f52abbe497de242c6ff 2024-11-13T05:39:56,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/5f893eee04fb424ca62d15b8a2038564 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5f893eee04fb424ca62d15b8a2038564 2024-11-13T05:39:56,991 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5f893eee04fb424ca62d15b8a2038564, entries=150, sequenceid=269, filesize=12.0 K 2024-11-13T05:39:56,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/b11fbe021c544a8f8b4766b6dcfbfa38 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/b11fbe021c544a8f8b4766b6dcfbfa38 2024-11-13T05:39:57,009 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/b11fbe021c544a8f8b4766b6dcfbfa38, entries=150, sequenceid=269, filesize=12.0 K 2024-11-13T05:39:57,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/38ad84b2e2784f52abbe497de242c6ff as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/38ad84b2e2784f52abbe497de242c6ff 2024-11-13T05:39:57,023 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/38ad84b2e2784f52abbe497de242c6ff, entries=150, sequenceid=269, filesize=12.0 K 2024-11-13T05:39:57,025 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 1385ms, sequenceid=269, compaction requested=true 2024-11-13T05:39:57,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:57,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:57,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-13T05:39:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-13T05:39:57,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-13T05:39:57,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5470 sec 2024-11-13T05:39:57,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.5630 sec 2024-11-13T05:39:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:57,242 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:57,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-13T05:39:57,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/841290a47247422d856a67be1a46e3dc is 50, key is test_row_0/A:col10/1731476397237/Put/seqid=0 2024-11-13T05:39:57,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476457257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:57,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476457265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:57,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476457267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:57,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476457269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:57,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476457269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:57,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:57,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741894_1070 (size=12301) 2024-11-13T05:39:57,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476457376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:57,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:57,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476457387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:57,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:57,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476457387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:57,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476457586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:57,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:57,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476457597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:57,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476457606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:39:57,637 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:39:57,637 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:39:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-13T05:39:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T05:39:57,640 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:39:57,641 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:39:57,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:39:57,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/841290a47247422d856a67be1a46e3dc 2024-11-13T05:39:57,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/0a1aed12ba324bcbb78f7569f5130310 is 50, key is test_row_0/B:col10/1731476397237/Put/seqid=0 2024-11-13T05:39:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T05:39:57,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741895_1071 (size=12301) 2024-11-13T05:39:57,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/0a1aed12ba324bcbb78f7569f5130310 2024-11-13T05:39:57,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/5cc69261809b4c97a7f1cd62dd0aa3dc is 50, key is test_row_0/C:col10/1731476397237/Put/seqid=0 2024-11-13T05:39:57,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:39:57,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:57,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:57,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:57,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:57,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:57,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741896_1072 (size=12301) 2024-11-13T05:39:57,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/5cc69261809b4c97a7f1cd62dd0aa3dc 2024-11-13T05:39:57,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/841290a47247422d856a67be1a46e3dc as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/841290a47247422d856a67be1a46e3dc 2024-11-13T05:39:57,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/841290a47247422d856a67be1a46e3dc, entries=150, sequenceid=297, filesize=12.0 K 2024-11-13T05:39:57,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/0a1aed12ba324bcbb78f7569f5130310 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0a1aed12ba324bcbb78f7569f5130310 2024-11-13T05:39:57,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0a1aed12ba324bcbb78f7569f5130310, entries=150, sequenceid=297, filesize=12.0 K 2024-11-13T05:39:57,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/5cc69261809b4c97a7f1cd62dd0aa3dc as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5cc69261809b4c97a7f1cd62dd0aa3dc 2024-11-13T05:39:57,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476457896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:57,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5cc69261809b4c97a7f1cd62dd0aa3dc, entries=150, sequenceid=297, filesize=12.0 K 2024-11-13T05:39:57,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 669ms, sequenceid=297, compaction requested=true 2024-11-13T05:39:57,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:57,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:39:57,911 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:39:57,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:57,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:39:57,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:57,911 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:39:57,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:39:57,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:39:57,913 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54280 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:39:57,913 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/A is initiating minor compaction (all files) 2024-11-13T05:39:57,913 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/A in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:57,914 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49500 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:39:57,914 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/B is initiating minor compaction (all files) 2024-11-13T05:39:57,914 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/B in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:57,915 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1c12e7ae8671427d849746ea5729a4a2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/7985a8244c5145809993c562e36c4111, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/b11fbe021c544a8f8b4766b6dcfbfa38, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0a1aed12ba324bcbb78f7569f5130310] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=48.3 K 2024-11-13T05:39:57,915 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c12e7ae8671427d849746ea5729a4a2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731476394063 2024-11-13T05:39:57,916 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 7985a8244c5145809993c562e36c4111, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731476394501 2024-11-13T05:39:57,916 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting b11fbe021c544a8f8b4766b6dcfbfa38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731476394876 2024-11-13T05:39:57,917 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a1aed12ba324bcbb78f7569f5130310, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731476396073 2024-11-13T05:39:57,918 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ec92832c4e9c4404ae614285a7f31d5f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5bd931cfa250485fb861377f24d9a918, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5f893eee04fb424ca62d15b8a2038564, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/841290a47247422d856a67be1a46e3dc] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=53.0 K 2024-11-13T05:39:57,919 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec92832c4e9c4404ae614285a7f31d5f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731476394063 2024-11-13T05:39:57,920 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bd931cfa250485fb861377f24d9a918, keycount=250, bloomtype=ROW, size=16.6 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731476394501 2024-11-13T05:39:57,920 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f893eee04fb424ca62d15b8a2038564, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731476394876 2024-11-13T05:39:57,921 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 841290a47247422d856a67be1a46e3dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731476396073 2024-11-13T05:39:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:57,931 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:57,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:39:57,948 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#B#compaction#60 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:57,949 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/302e7c58657a4b2795a3671f7979d3f7 is 50, key is test_row_0/B:col10/1731476397237/Put/seqid=0 2024-11-13T05:39:57,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:39:57,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:57,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:57,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:57,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:57,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:57,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T05:39:57,959 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#A#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:57,960 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/ce554876c6e84fe38e13b7b6eb6bcbae is 50, key is test_row_0/A:col10/1731476397237/Put/seqid=0 2024-11-13T05:39:57,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:57,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/13f1b0021cb2486d9e47a247133fba6b is 50, key is test_row_0/A:col10/1731476397919/Put/seqid=0 2024-11-13T05:39:58,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741897_1073 (size=12983) 2024-11-13T05:39:58,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741898_1074 (size=12983) 2024-11-13T05:39:58,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741899_1075 (size=14741) 2024-11-13T05:39:58,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/13f1b0021cb2486d9e47a247133fba6b 2024-11-13T05:39:58,049 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/ce554876c6e84fe38e13b7b6eb6bcbae as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ce554876c6e84fe38e13b7b6eb6bcbae 2024-11-13T05:39:58,063 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/A of 6abd824d4b7a94f2d19ab3e39e94ccfe into ce554876c6e84fe38e13b7b6eb6bcbae(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:58,063 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:58,063 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/A, priority=12, startTime=1731476397911; duration=0sec 2024-11-13T05:39:58,064 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:39:58,064 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:A 2024-11-13T05:39:58,064 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:39:58,066 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49500 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:39:58,066 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/C is initiating minor compaction (all files) 2024-11-13T05:39:58,067 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/C in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:58,067 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/c3c84ebdae074927935dfd0f3c83ff3e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/945d45fc464d48d2ba913fe2f8c34cd8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/38ad84b2e2784f52abbe497de242c6ff, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5cc69261809b4c97a7f1cd62dd0aa3dc] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=48.3 K 2024-11-13T05:39:58,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:58,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476458053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:58,068 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3c84ebdae074927935dfd0f3c83ff3e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731476394063 2024-11-13T05:39:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:58,068 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 945d45fc464d48d2ba913fe2f8c34cd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731476394501 2024-11-13T05:39:58,069 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38ad84b2e2784f52abbe497de242c6ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731476394876 2024-11-13T05:39:58,069 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cc69261809b4c97a7f1cd62dd0aa3dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731476396073 2024-11-13T05:39:58,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/bda5ba4388f240a282cff85b76e43032 is 50, key is test_row_0/B:col10/1731476397919/Put/seqid=0 2024-11-13T05:39:58,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476458068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:58,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:58,093 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#C#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:39:58,096 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/8c2902a6b58f424eb509dd51cbfdca73 is 50, key is test_row_0/C:col10/1731476397237/Put/seqid=0 2024-11-13T05:39:58,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741900_1076 (size=12301) 2024-11-13T05:39:58,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:39:58,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:58,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:58,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:58,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741901_1077 (size=12983) 2024-11-13T05:39:58,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:58,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476458177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:58,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476458186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T05:39:58,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:39:58,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:58,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:58,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:58,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476458287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:58,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476458287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:58,300 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in 6abd824d4b7a94f2d19ab3e39e94ccfe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:39:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:58,300 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in 6abd824d4b7a94f2d19ab3e39e94ccfe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:39:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:58,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476458387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:58,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:58,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476458396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:58,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:58,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476458408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:58,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:39:58,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:58,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:58,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:58,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,432 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/302e7c58657a4b2795a3671f7979d3f7 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/302e7c58657a4b2795a3671f7979d3f7 2024-11-13T05:39:58,461 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/B of 6abd824d4b7a94f2d19ab3e39e94ccfe into 302e7c58657a4b2795a3671f7979d3f7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:58,461 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:58,461 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/B, priority=12, startTime=1731476397911; duration=0sec 2024-11-13T05:39:58,461 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:58,461 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:B 2024-11-13T05:39:58,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/bda5ba4388f240a282cff85b76e43032 2024-11-13T05:39:58,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/34ffaefbc14a4bc8ac4eda1f66d7a833 is 50, key is test_row_0/C:col10/1731476397919/Put/seqid=0 2024-11-13T05:39:58,547 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/8c2902a6b58f424eb509dd51cbfdca73 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/8c2902a6b58f424eb509dd51cbfdca73 2024-11-13T05:39:58,556 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/C of 6abd824d4b7a94f2d19ab3e39e94ccfe into 8c2902a6b58f424eb509dd51cbfdca73(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:39:58,556 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:58,556 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/C, priority=12, startTime=1731476397911; duration=0sec 2024-11-13T05:39:58,556 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:39:58,556 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:C 2024-11-13T05:39:58,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741902_1078 (size=12301) 2024-11-13T05:39:58,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:39:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:58,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476458696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:58,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:58,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476458707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:58,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:58,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:58,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:39:58,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:58,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:58,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:58,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T05:39:58,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:39:58,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:58,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:58,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:58,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:58,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/34ffaefbc14a4bc8ac4eda1f66d7a833 2024-11-13T05:39:58,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/13f1b0021cb2486d9e47a247133fba6b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/13f1b0021cb2486d9e47a247133fba6b 2024-11-13T05:39:58,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/13f1b0021cb2486d9e47a247133fba6b, entries=200, sequenceid=309, filesize=14.4 K 2024-11-13T05:39:58,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/bda5ba4388f240a282cff85b76e43032 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/bda5ba4388f240a282cff85b76e43032 2024-11-13T05:39:59,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/bda5ba4388f240a282cff85b76e43032, entries=150, sequenceid=309, filesize=12.0 K 2024-11-13T05:39:59,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/34ffaefbc14a4bc8ac4eda1f66d7a833 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/34ffaefbc14a4bc8ac4eda1f66d7a833 2024-11-13T05:39:59,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/34ffaefbc14a4bc8ac4eda1f66d7a833, entries=150, sequenceid=309, filesize=12.0 K 2024-11-13T05:39:59,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 1088ms, sequenceid=309, compaction requested=false 2024-11-13T05:39:59,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:59,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:39:59,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:59,042 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T05:39:59,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/5cab3b4fff9b467898972204168f84a4 is 50, key is test_row_0/A:col10/1731476398048/Put/seqid=0 2024-11-13T05:39:59,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741903_1079 (size=12301) 2024-11-13T05:39:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:59,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:59,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476459227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:59,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476459229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:59,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476459336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:59,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:59,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476459336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:59,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:59,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476459426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:59,427 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in 6abd824d4b7a94f2d19ab3e39e94ccfe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:39:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:59,480 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/5cab3b4fff9b467898972204168f84a4 2024-11-13T05:39:59,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/6db953a90dcb47cf9a114797c8284ea4 is 50, key is test_row_0/B:col10/1731476398048/Put/seqid=0 2024-11-13T05:39:59,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741904_1080 (size=12301) 2024-11-13T05:39:59,503 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/6db953a90dcb47cf9a114797c8284ea4 2024-11-13T05:39:59,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/9dab42c13fb34a2f9304bb1a69cdf34e is 50, key is test_row_0/C:col10/1731476398048/Put/seqid=0 2024-11-13T05:39:59,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741905_1081 (size=12301) 2024-11-13T05:39:59,543 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/9dab42c13fb34a2f9304bb1a69cdf34e 2024-11-13T05:39:59,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476459546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:59,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476459547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:59,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:59,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:59,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/5cab3b4fff9b467898972204168f84a4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5cab3b4fff9b467898972204168f84a4 2024-11-13T05:39:59,562 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5cab3b4fff9b467898972204168f84a4, entries=150, sequenceid=336, filesize=12.0 K 2024-11-13T05:39:59,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/6db953a90dcb47cf9a114797c8284ea4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/6db953a90dcb47cf9a114797c8284ea4 2024-11-13T05:39:59,579 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/6db953a90dcb47cf9a114797c8284ea4, entries=150, sequenceid=336, filesize=12.0 K 2024-11-13T05:39:59,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/9dab42c13fb34a2f9304bb1a69cdf34e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/9dab42c13fb34a2f9304bb1a69cdf34e 2024-11-13T05:39:59,591 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/9dab42c13fb34a2f9304bb1a69cdf34e, entries=150, sequenceid=336, filesize=12.0 K 2024-11-13T05:39:59,593 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 551ms, sequenceid=336, compaction requested=true 2024-11-13T05:39:59,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:39:59,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:59,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-13T05:39:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-13T05:39:59,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-13T05:39:59,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9540 sec 2024-11-13T05:39:59,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.9620 sec 2024-11-13T05:39:59,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T05:39:59,797 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:39:59,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:39:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-13T05:39:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T05:39:59,801 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:39:59,802 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:39:59,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:39:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:39:59,859 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:39:59,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T05:39:59,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/8edf8c0e9c92477c8983792aa2e1b6c2 is 50, key is test_row_0/A:col10/1731476399858/Put/seqid=0 2024-11-13T05:39:59,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741906_1082 (size=17181) 2024-11-13T05:39:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T05:39:59,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476459933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:59,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:39:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476459934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:39:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:39:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:39:59,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:39:59,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:59,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:39:59,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:39:59,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:59,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:39:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476460046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:00,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476460047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:00,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:00,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:40:00,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:00,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:00,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:00,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T05:40:00,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476460257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:00,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:00,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:40:00,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:00,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:00,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:00,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476460266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:00,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:00,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/8edf8c0e9c92477c8983792aa2e1b6c2 2024-11-13T05:40:00,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/b627f2e4984e4905b94214c83699fdc6 is 50, key is test_row_0/B:col10/1731476399858/Put/seqid=0 2024-11-13T05:40:00,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476460317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:00,321 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in 6abd824d4b7a94f2d19ab3e39e94ccfe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:00,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476460326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:00,334 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in 6abd824d4b7a94f2d19ab3e39e94ccfe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:00,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741907_1083 (size=12301) 2024-11-13T05:40:00,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:40:00,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:00,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:00,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:00,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T05:40:00,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476460567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:00,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:40:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:00,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476460576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:00,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:40:00,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:00,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:00,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:00,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/b627f2e4984e4905b94214c83699fdc6 2024-11-13T05:40:00,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/abf81e11856f4306a0f8cfc6a2672c02 is 50, key is test_row_0/C:col10/1731476399858/Put/seqid=0 2024-11-13T05:40:00,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741908_1084 (size=12301) 2024-11-13T05:40:00,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:40:00,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:00,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:00,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:00,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T05:40:01,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:40:01,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:01,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:01,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:01,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:01,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:01,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476461076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:01,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476461086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:01,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/abf81e11856f4306a0f8cfc6a2672c02 2024-11-13T05:40:01,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/8edf8c0e9c92477c8983792aa2e1b6c2 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8edf8c0e9c92477c8983792aa2e1b6c2 2024-11-13T05:40:01,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8edf8c0e9c92477c8983792aa2e1b6c2, entries=250, sequenceid=350, filesize=16.8 K 2024-11-13T05:40:01,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/b627f2e4984e4905b94214c83699fdc6 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/b627f2e4984e4905b94214c83699fdc6 2024-11-13T05:40:01,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/b627f2e4984e4905b94214c83699fdc6, entries=150, sequenceid=350, filesize=12.0 K 2024-11-13T05:40:01,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/abf81e11856f4306a0f8cfc6a2672c02 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/abf81e11856f4306a0f8cfc6a2672c02 2024-11-13T05:40:01,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:40:01,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:01,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:01,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:01,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:01,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:01,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/abf81e11856f4306a0f8cfc6a2672c02, entries=150, sequenceid=350, filesize=12.0 K 2024-11-13T05:40:01,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 1341ms, sequenceid=350, compaction requested=true 2024-11-13T05:40:01,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:01,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:01,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:01,201 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:40:01,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:01,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:01,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:01,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T05:40:01,201 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:40:01,203 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:40:01,203 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/B is initiating minor compaction (all files) 2024-11-13T05:40:01,203 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/B in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:01,203 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/302e7c58657a4b2795a3671f7979d3f7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/bda5ba4388f240a282cff85b76e43032, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/6db953a90dcb47cf9a114797c8284ea4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/b627f2e4984e4905b94214c83699fdc6] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=48.7 K 2024-11-13T05:40:01,204 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 302e7c58657a4b2795a3671f7979d3f7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731476396073 2024-11-13T05:40:01,204 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting bda5ba4388f240a282cff85b76e43032, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731476397264 2024-11-13T05:40:01,204 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57206 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:40:01,205 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/A is initiating minor compaction (all files) 2024-11-13T05:40:01,205 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/A in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:01,205 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ce554876c6e84fe38e13b7b6eb6bcbae, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/13f1b0021cb2486d9e47a247133fba6b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5cab3b4fff9b467898972204168f84a4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8edf8c0e9c92477c8983792aa2e1b6c2] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=55.9 K 2024-11-13T05:40:01,205 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6db953a90dcb47cf9a114797c8284ea4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731476398036 2024-11-13T05:40:01,205 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting ce554876c6e84fe38e13b7b6eb6bcbae, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731476396073 2024-11-13T05:40:01,205 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting b627f2e4984e4905b94214c83699fdc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731476399225 2024-11-13T05:40:01,205 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 13f1b0021cb2486d9e47a247133fba6b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731476397264 2024-11-13T05:40:01,206 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cab3b4fff9b467898972204168f84a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731476398036 2024-11-13T05:40:01,207 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 8edf8c0e9c92477c8983792aa2e1b6c2, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731476399219 2024-11-13T05:40:01,245 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#B#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:01,246 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/507c260bfac143aeae083ebd009b7de8 is 50, key is test_row_0/B:col10/1731476399858/Put/seqid=0 2024-11-13T05:40:01,254 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#A#compaction#73 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:01,254 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/4ea6a5aa6d8445a2b8defc8585224659 is 50, key is test_row_0/A:col10/1731476399858/Put/seqid=0 2024-11-13T05:40:01,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741910_1086 (size=13119) 2024-11-13T05:40:01,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741909_1085 (size=13119) 2024-11-13T05:40:01,325 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/4ea6a5aa6d8445a2b8defc8585224659 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/4ea6a5aa6d8445a2b8defc8585224659 2024-11-13T05:40:01,337 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/A of 6abd824d4b7a94f2d19ab3e39e94ccfe into 4ea6a5aa6d8445a2b8defc8585224659(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:01,338 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:01,338 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/A, priority=12, startTime=1731476401201; duration=0sec 2024-11-13T05:40:01,338 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:01,338 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:A 2024-11-13T05:40:01,338 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:40:01,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:40:01,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:01,343 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:40:01,343 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T05:40:01,343 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/C is initiating minor compaction (all files) 2024-11-13T05:40:01,343 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/C in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:01,343 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/8c2902a6b58f424eb509dd51cbfdca73, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/34ffaefbc14a4bc8ac4eda1f66d7a833, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/9dab42c13fb34a2f9304bb1a69cdf34e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/abf81e11856f4306a0f8cfc6a2672c02] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=48.7 K 2024-11-13T05:40:01,344 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c2902a6b58f424eb509dd51cbfdca73, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731476396073 2024-11-13T05:40:01,346 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 34ffaefbc14a4bc8ac4eda1f66d7a833, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731476397264 2024-11-13T05:40:01,346 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dab42c13fb34a2f9304bb1a69cdf34e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731476398036 2024-11-13T05:40:01,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/a13a026f038e4a4a826087ce4d989b30 is 50, key is test_row_0/A:col10/1731476399924/Put/seqid=0 2024-11-13T05:40:01,363 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting abf81e11856f4306a0f8cfc6a2672c02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731476399225 2024-11-13T05:40:01,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741911_1087 (size=12301) 2024-11-13T05:40:01,386 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#C#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:01,388 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/fd49f4bc3837435c8dc6c2a28251c265 is 50, key is test_row_0/C:col10/1731476399858/Put/seqid=0 2024-11-13T05:40:01,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741912_1088 (size=13119) 2024-11-13T05:40:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:40:01,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:01,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476461502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:01,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476461617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:01,722 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/507c260bfac143aeae083ebd009b7de8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/507c260bfac143aeae083ebd009b7de8 2024-11-13T05:40:01,735 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/B of 6abd824d4b7a94f2d19ab3e39e94ccfe into 507c260bfac143aeae083ebd009b7de8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:01,735 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:01,735 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/B, priority=12, startTime=1731476401201; duration=0sec 2024-11-13T05:40:01,735 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:01,735 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:B 2024-11-13T05:40:01,782 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/a13a026f038e4a4a826087ce4d989b30 2024-11-13T05:40:01,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/f7478b406efb4c77bda53cde272773f6 is 50, key is test_row_0/B:col10/1731476399924/Put/seqid=0 2024-11-13T05:40:01,817 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/fd49f4bc3837435c8dc6c2a28251c265 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/fd49f4bc3837435c8dc6c2a28251c265 2024-11-13T05:40:01,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476461827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:01,835 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/C of 6abd824d4b7a94f2d19ab3e39e94ccfe into fd49f4bc3837435c8dc6c2a28251c265(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:01,835 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:01,835 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/C, priority=12, startTime=1731476401201; duration=0sec 2024-11-13T05:40:01,836 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:01,836 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:C 2024-11-13T05:40:01,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741913_1089 (size=12301) 2024-11-13T05:40:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T05:40:02,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:02,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476462097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476462097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:02,101 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_0' in 6abd824d4b7a94f2d19ab3e39e94ccfe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:02,101 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in 6abd824d4b7a94f2d19ab3e39e94ccfe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:02,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:02,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:02,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:02,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:02,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:02,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:02,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476462136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:02,250 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/f7478b406efb4c77bda53cde272773f6 2024-11-13T05:40:02,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/c99d446553424c5190bd2c8b96d25ba2 is 50, key is test_row_0/C:col10/1731476399924/Put/seqid=0 2024-11-13T05:40:02,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741914_1090 (size=12301) 2024-11-13T05:40:02,270 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/c99d446553424c5190bd2c8b96d25ba2 2024-11-13T05:40:02,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/a13a026f038e4a4a826087ce4d989b30 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/a13a026f038e4a4a826087ce4d989b30 2024-11-13T05:40:02,285 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/a13a026f038e4a4a826087ce4d989b30, entries=150, sequenceid=374, filesize=12.0 K 2024-11-13T05:40:02,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/f7478b406efb4c77bda53cde272773f6 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f7478b406efb4c77bda53cde272773f6 2024-11-13T05:40:02,292 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f7478b406efb4c77bda53cde272773f6, entries=150, sequenceid=374, filesize=12.0 K 2024-11-13T05:40:02,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/c99d446553424c5190bd2c8b96d25ba2 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/c99d446553424c5190bd2c8b96d25ba2 2024-11-13T05:40:02,303 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/c99d446553424c5190bd2c8b96d25ba2, entries=150, sequenceid=374, filesize=12.0 K 2024-11-13T05:40:02,305 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 962ms, sequenceid=374, compaction requested=false 2024-11-13T05:40:02,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:02,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:02,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-13T05:40:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-13T05:40:02,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T05:40:02,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5070 sec 2024-11-13T05:40:02,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.5150 sec 2024-11-13T05:40:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:40:02,649 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:02,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T05:40:02,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/13681015645140c1a26ff201c74dd7ea is 50, key is test_row_0/A:col10/1731476402647/Put/seqid=0 2024-11-13T05:40:02,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741915_1091 (size=14741) 2024-11-13T05:40:02,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/13681015645140c1a26ff201c74dd7ea 2024-11-13T05:40:02,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/053dd5c872474a44bc93ec1603a031fd is 50, key is test_row_0/B:col10/1731476402647/Put/seqid=0 2024-11-13T05:40:02,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741916_1092 (size=12301) 2024-11-13T05:40:02,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/053dd5c872474a44bc93ec1603a031fd 2024-11-13T05:40:02,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/e89af198f73f44b5a3fe9a65d472dbc5 is 50, key is test_row_0/C:col10/1731476402647/Put/seqid=0 2024-11-13T05:40:02,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476462766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:02,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741917_1093 (size=12301) 2024-11-13T05:40:02,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476462877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:03,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476463086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:03,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:03,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:03,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:03,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/e89af198f73f44b5a3fe9a65d472dbc5 2024-11-13T05:40:03,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/13681015645140c1a26ff201c74dd7ea as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/13681015645140c1a26ff201c74dd7ea 2024-11-13T05:40:03,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/13681015645140c1a26ff201c74dd7ea, entries=200, sequenceid=390, filesize=14.4 K 2024-11-13T05:40:03,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/053dd5c872474a44bc93ec1603a031fd as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/053dd5c872474a44bc93ec1603a031fd 2024-11-13T05:40:03,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/053dd5c872474a44bc93ec1603a031fd, entries=150, sequenceid=390, filesize=12.0 K 2024-11-13T05:40:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/e89af198f73f44b5a3fe9a65d472dbc5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/e89af198f73f44b5a3fe9a65d472dbc5 2024-11-13T05:40:03,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/e89af198f73f44b5a3fe9a65d472dbc5, entries=150, sequenceid=390, filesize=12.0 K 2024-11-13T05:40:03,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 554ms, sequenceid=390, compaction requested=true 2024-11-13T05:40:03,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:03,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:03,203 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:03,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:03,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:03,204 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:03,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:03,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:03,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:03,205 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:03,205 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:03,205 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/B is initiating minor compaction (all files) 2024-11-13T05:40:03,205 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/A is initiating minor compaction (all files) 2024-11-13T05:40:03,205 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/B in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:03,205 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/A in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:03,205 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/507c260bfac143aeae083ebd009b7de8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f7478b406efb4c77bda53cde272773f6, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/053dd5c872474a44bc93ec1603a031fd] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=36.8 K 2024-11-13T05:40:03,205 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/4ea6a5aa6d8445a2b8defc8585224659, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/a13a026f038e4a4a826087ce4d989b30, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/13681015645140c1a26ff201c74dd7ea] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=39.2 K 2024-11-13T05:40:03,206 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ea6a5aa6d8445a2b8defc8585224659, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731476399225 2024-11-13T05:40:03,206 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 507c260bfac143aeae083ebd009b7de8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731476399225 2024-11-13T05:40:03,206 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting a13a026f038e4a4a826087ce4d989b30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731476399924 2024-11-13T05:40:03,206 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting f7478b406efb4c77bda53cde272773f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731476399924 2024-11-13T05:40:03,207 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 053dd5c872474a44bc93ec1603a031fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731476401449 2024-11-13T05:40:03,207 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13681015645140c1a26ff201c74dd7ea, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731476401449 2024-11-13T05:40:03,220 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#A#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:03,221 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/12562f6d50bc4430abf2e44a057d1451 is 50, key is test_row_0/A:col10/1731476402647/Put/seqid=0 2024-11-13T05:40:03,223 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#B#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:03,223 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/eb7b81d812094c63a61cdbabdcb07f54 is 50, key is test_row_0/B:col10/1731476402647/Put/seqid=0 2024-11-13T05:40:03,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741918_1094 (size=13221) 2024-11-13T05:40:03,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741919_1095 (size=13221) 2024-11-13T05:40:03,258 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/eb7b81d812094c63a61cdbabdcb07f54 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/eb7b81d812094c63a61cdbabdcb07f54 2024-11-13T05:40:03,267 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/B of 6abd824d4b7a94f2d19ab3e39e94ccfe into eb7b81d812094c63a61cdbabdcb07f54(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:03,267 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:03,267 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/B, priority=13, startTime=1731476403204; duration=0sec 2024-11-13T05:40:03,267 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:03,267 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:B 2024-11-13T05:40:03,267 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:03,269 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:03,269 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/C is initiating minor compaction (all files) 2024-11-13T05:40:03,269 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/C in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:03,269 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/fd49f4bc3837435c8dc6c2a28251c265, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/c99d446553424c5190bd2c8b96d25ba2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/e89af198f73f44b5a3fe9a65d472dbc5] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=36.8 K 2024-11-13T05:40:03,270 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting fd49f4bc3837435c8dc6c2a28251c265, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731476399225 2024-11-13T05:40:03,270 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting c99d446553424c5190bd2c8b96d25ba2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731476399924 2024-11-13T05:40:03,271 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting e89af198f73f44b5a3fe9a65d472dbc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731476401449 2024-11-13T05:40:03,283 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#C#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:03,283 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/3030c7a9c4c243c0be23632cc94e8f4f is 50, key is test_row_0/C:col10/1731476402647/Put/seqid=0 2024-11-13T05:40:03,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741920_1096 (size=13221) 2024-11-13T05:40:03,303 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/3030c7a9c4c243c0be23632cc94e8f4f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/3030c7a9c4c243c0be23632cc94e8f4f 2024-11-13T05:40:03,312 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/C of 6abd824d4b7a94f2d19ab3e39e94ccfe into 3030c7a9c4c243c0be23632cc94e8f4f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:03,312 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:03,312 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/C, priority=13, startTime=1731476403204; duration=0sec 2024-11-13T05:40:03,313 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:03,313 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:C 2024-11-13T05:40:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:40:03,398 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:03,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T05:40:03,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/943120eaeba9425a8525f31d86b4cde5 is 50, key is test_row_0/A:col10/1731476402754/Put/seqid=0 2024-11-13T05:40:03,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741921_1097 (size=12301) 2024-11-13T05:40:03,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/943120eaeba9425a8525f31d86b4cde5 2024-11-13T05:40:03,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476463442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:03,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/0d27e79389214b80a6fa858d1f8a54bc is 50, key is test_row_0/B:col10/1731476402754/Put/seqid=0 2024-11-13T05:40:03,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741922_1098 (size=12301) 2024-11-13T05:40:03,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/0d27e79389214b80a6fa858d1f8a54bc 2024-11-13T05:40:03,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/62c5d890c8a54198a77e3359c67ea8b0 is 50, key is test_row_0/C:col10/1731476402754/Put/seqid=0 2024-11-13T05:40:03,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741923_1099 (size=12301) 2024-11-13T05:40:03,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476463557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:03,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:03,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:03,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:03,657 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/12562f6d50bc4430abf2e44a057d1451 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/12562f6d50bc4430abf2e44a057d1451 2024-11-13T05:40:03,676 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/A of 6abd824d4b7a94f2d19ab3e39e94ccfe into 12562f6d50bc4430abf2e44a057d1451(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:03,676 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:03,676 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/A, priority=13, startTime=1731476403203; duration=0sec 2024-11-13T05:40:03,676 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:03,676 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:A 2024-11-13T05:40:03,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476463767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:03,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/62c5d890c8a54198a77e3359c67ea8b0 2024-11-13T05:40:03,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/943120eaeba9425a8525f31d86b4cde5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/943120eaeba9425a8525f31d86b4cde5 2024-11-13T05:40:03,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/943120eaeba9425a8525f31d86b4cde5, entries=150, sequenceid=415, filesize=12.0 K 2024-11-13T05:40:03,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/0d27e79389214b80a6fa858d1f8a54bc as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0d27e79389214b80a6fa858d1f8a54bc 2024-11-13T05:40:03,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0d27e79389214b80a6fa858d1f8a54bc, entries=150, sequenceid=415, filesize=12.0 K 2024-11-13T05:40:03,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/62c5d890c8a54198a77e3359c67ea8b0 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/62c5d890c8a54198a77e3359c67ea8b0 2024-11-13T05:40:03,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/62c5d890c8a54198a77e3359c67ea8b0, entries=150, sequenceid=415, filesize=12.0 K 2024-11-13T05:40:03,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 541ms, sequenceid=415, compaction requested=false 2024-11-13T05:40:03,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:03,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T05:40:03,969 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:03,969 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:03,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-13T05:40:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T05:40:03,972 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:03,973 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:03,973 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T05:40:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:40:04,093 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:04,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T05:40:04,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/911fc67f089b45c9ae8a65ec30b321f6 is 50, key is test_row_0/A:col10/1731476404084/Put/seqid=0 2024-11-13T05:40:04,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T05:40:04,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:04,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:04,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:04,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:04,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:04,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741924_1100 (size=12301) 2024-11-13T05:40:04,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/911fc67f089b45c9ae8a65ec30b321f6 2024-11-13T05:40:04,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/9e00f6f28d8d40498c8c3861e746c0c9 is 50, key is test_row_0/B:col10/1731476404084/Put/seqid=0 2024-11-13T05:40:04,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476464173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476464175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476464176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741925_1101 (size=12301) 2024-11-13T05:40:04,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T05:40:04,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:04,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:04,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:04,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:04,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T05:40:04,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476464287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476464286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476464296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42958 deadline: 1731476464336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,337 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in 6abd824d4b7a94f2d19ab3e39e94ccfe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42948 deadline: 1731476464377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,378 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in 6abd824d4b7a94f2d19ab3e39e94ccfe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T05:40:04,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:04,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:04,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:04,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:04,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476464496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476464497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476464506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T05:40:04,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T05:40:04,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:04,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:04,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:04,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:04,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:04,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/9e00f6f28d8d40498c8c3861e746c0c9 2024-11-13T05:40:04,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/636673c08c57483383c6b9cad82bc91e is 50, key is test_row_0/C:col10/1731476404084/Put/seqid=0 2024-11-13T05:40:04,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741926_1102 (size=12301) 2024-11-13T05:40:04,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/636673c08c57483383c6b9cad82bc91e 2024-11-13T05:40:04,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/911fc67f089b45c9ae8a65ec30b321f6 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/911fc67f089b45c9ae8a65ec30b321f6 2024-11-13T05:40:04,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/911fc67f089b45c9ae8a65ec30b321f6, entries=150, sequenceid=430, filesize=12.0 K 2024-11-13T05:40:04,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/9e00f6f28d8d40498c8c3861e746c0c9 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/9e00f6f28d8d40498c8c3861e746c0c9 2024-11-13T05:40:04,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/9e00f6f28d8d40498c8c3861e746c0c9, entries=150, sequenceid=430, filesize=12.0 K 2024-11-13T05:40:04,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/636673c08c57483383c6b9cad82bc91e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/636673c08c57483383c6b9cad82bc91e 2024-11-13T05:40:04,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/636673c08c57483383c6b9cad82bc91e, entries=150, sequenceid=430, filesize=12.0 K 2024-11-13T05:40:04,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 629ms, sequenceid=430, compaction requested=true 2024-11-13T05:40:04,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:04,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:04,723 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:04,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:04,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:04,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:04,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:04,725 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:04,725 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:04,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T05:40:04,725 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/A is initiating minor compaction (all files) 2024-11-13T05:40:04,725 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/A in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:04,725 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/12562f6d50bc4430abf2e44a057d1451, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/943120eaeba9425a8525f31d86b4cde5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/911fc67f089b45c9ae8a65ec30b321f6] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=36.9 K 2024-11-13T05:40:04,725 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12562f6d50bc4430abf2e44a057d1451, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731476401449 2024-11-13T05:40:04,727 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 943120eaeba9425a8525f31d86b4cde5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731476402754 2024-11-13T05:40:04,728 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 911fc67f089b45c9ae8a65ec30b321f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731476403426 2024-11-13T05:40:04,729 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:04,729 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/B is initiating minor compaction (all files) 2024-11-13T05:40:04,729 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/B in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:04,729 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/eb7b81d812094c63a61cdbabdcb07f54, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0d27e79389214b80a6fa858d1f8a54bc, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/9e00f6f28d8d40498c8c3861e746c0c9] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=36.9 K 2024-11-13T05:40:04,731 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting eb7b81d812094c63a61cdbabdcb07f54, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731476401449 2024-11-13T05:40:04,731 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d27e79389214b80a6fa858d1f8a54bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731476402754 2024-11-13T05:40:04,733 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e00f6f28d8d40498c8c3861e746c0c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731476403426 2024-11-13T05:40:04,744 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:04,744 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/313eeae9c0ec42d09d6606c8b9a5bbe7 is 50, key is test_row_0/A:col10/1731476404084/Put/seqid=0 2024-11-13T05:40:04,753 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:04,754 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/66173a37292c473b989fcb48318fb235 is 50, key is test_row_0/B:col10/1731476404084/Put/seqid=0 2024-11-13T05:40:04,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T05:40:04,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:04,760 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T05:40:04,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741928_1104 (size=13323) 2024-11-13T05:40:04,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/b05196bfb0114172aad931ae62621e45 is 50, key is test_row_0/A:col10/1731476404136/Put/seqid=0 2024-11-13T05:40:04,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741927_1103 (size=13323) 2024-11-13T05:40:04,803 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/66173a37292c473b989fcb48318fb235 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/66173a37292c473b989fcb48318fb235 2024-11-13T05:40:04,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:40:04,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:04,819 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/313eeae9c0ec42d09d6606c8b9a5bbe7 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/313eeae9c0ec42d09d6606c8b9a5bbe7 2024-11-13T05:40:04,820 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/B of 6abd824d4b7a94f2d19ab3e39e94ccfe into 66173a37292c473b989fcb48318fb235(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:04,820 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:04,820 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/B, priority=13, startTime=1731476404723; duration=0sec 2024-11-13T05:40:04,821 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:04,821 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:B 2024-11-13T05:40:04,821 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:04,823 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:04,823 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/C is initiating minor compaction (all files) 2024-11-13T05:40:04,823 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/C in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:04,823 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/3030c7a9c4c243c0be23632cc94e8f4f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/62c5d890c8a54198a77e3359c67ea8b0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/636673c08c57483383c6b9cad82bc91e] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=36.9 K 2024-11-13T05:40:04,824 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 3030c7a9c4c243c0be23632cc94e8f4f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731476401449 2024-11-13T05:40:04,824 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 62c5d890c8a54198a77e3359c67ea8b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731476402754 2024-11-13T05:40:04,825 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 636673c08c57483383c6b9cad82bc91e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731476403426 2024-11-13T05:40:04,830 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/A of 6abd824d4b7a94f2d19ab3e39e94ccfe into 313eeae9c0ec42d09d6606c8b9a5bbe7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:04,830 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:04,830 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/A, priority=13, startTime=1731476404723; duration=0sec 2024-11-13T05:40:04,830 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:04,830 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:A 2024-11-13T05:40:04,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741929_1105 (size=12301) 2024-11-13T05:40:04,834 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/b05196bfb0114172aad931ae62621e45 2024-11-13T05:40:04,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476464843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,857 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#C#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:04,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476464851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,857 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/a5b78118d84a4a6994789ea4f7bf0542 is 50, key is test_row_0/C:col10/1731476404084/Put/seqid=0 2024-11-13T05:40:04,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476464852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/e541eab3468f4cc69e184e4f7652f431 is 50, key is test_row_0/B:col10/1731476404136/Put/seqid=0 2024-11-13T05:40:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741931_1107 (size=13323) 2024-11-13T05:40:04,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741930_1106 (size=12301) 2024-11-13T05:40:04,912 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/a5b78118d84a4a6994789ea4f7bf0542 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/a5b78118d84a4a6994789ea4f7bf0542 2024-11-13T05:40:04,925 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/C of 6abd824d4b7a94f2d19ab3e39e94ccfe into a5b78118d84a4a6994789ea4f7bf0542(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:04,926 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:04,926 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/C, priority=13, startTime=1731476404724; duration=0sec 2024-11-13T05:40:04,926 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:04,926 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:C 2024-11-13T05:40:04,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476464956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476464966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:04,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476464967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T05:40:05,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476465166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:05,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476465176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:05,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476465186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:05,306 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/e541eab3468f4cc69e184e4f7652f431 2024-11-13T05:40:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/1b09a51fd34e49409e25a7304673bb01 is 50, key is test_row_0/C:col10/1731476404136/Put/seqid=0 2024-11-13T05:40:05,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741932_1108 (size=12301) 2024-11-13T05:40:05,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476465487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:05,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476465487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:05,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476465496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:05,748 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/1b09a51fd34e49409e25a7304673bb01 2024-11-13T05:40:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/b05196bfb0114172aad931ae62621e45 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/b05196bfb0114172aad931ae62621e45 2024-11-13T05:40:05,766 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/b05196bfb0114172aad931ae62621e45, entries=150, sequenceid=452, filesize=12.0 K 2024-11-13T05:40:05,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/e541eab3468f4cc69e184e4f7652f431 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/e541eab3468f4cc69e184e4f7652f431 2024-11-13T05:40:05,774 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/e541eab3468f4cc69e184e4f7652f431, entries=150, sequenceid=452, filesize=12.0 K 2024-11-13T05:40:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/1b09a51fd34e49409e25a7304673bb01 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/1b09a51fd34e49409e25a7304673bb01 2024-11-13T05:40:05,782 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/1b09a51fd34e49409e25a7304673bb01, entries=150, sequenceid=452, filesize=12.0 K 2024-11-13T05:40:05,787 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 1026ms, sequenceid=452, compaction requested=false 2024-11-13T05:40:05,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:05,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:05,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-13T05:40:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-13T05:40:05,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-13T05:40:05,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8190 sec 2024-11-13T05:40:05,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.8260 sec 2024-11-13T05:40:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:40:05,998 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:05,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T05:40:06,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/b9477331bb3d421dbea23ecf16e412ba is 50, key is test_row_0/A:col10/1731476405997/Put/seqid=0 2024-11-13T05:40:06,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741933_1109 (size=17181) 2024-11-13T05:40:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476466025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476466026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:06,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476466029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T05:40:06,117 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:06,117 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-13T05:40:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T05:40:06,119 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:06,120 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:06,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:06,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476466136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:06,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476466136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:06,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476466136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T05:40:06,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T05:40:06,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:06,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:06,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:06,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476466346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:06,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:06,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476466346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:06,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476466346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:06,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/b9477331bb3d421dbea23ecf16e412ba 2024-11-13T05:40:06,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/24f065da8a714c36992e7a5f00bffa46 is 50, key is test_row_0/B:col10/1731476405997/Put/seqid=0 2024-11-13T05:40:06,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T05:40:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:06,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741934_1110 (size=12301) 2024-11-13T05:40:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T05:40:06,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T05:40:06,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:06,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:06,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:06,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476466656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476466656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:06,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476466656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:06,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T05:40:06,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:06,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:06,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:06,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T05:40:06,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/24f065da8a714c36992e7a5f00bffa46 2024-11-13T05:40:06,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/a542fa2a1ebe4f748e888a25f7124944 is 50, key is test_row_0/C:col10/1731476405997/Put/seqid=0 2024-11-13T05:40:06,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741935_1111 (size=12301) 2024-11-13T05:40:06,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/a542fa2a1ebe4f748e888a25f7124944 2024-11-13T05:40:06,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/b9477331bb3d421dbea23ecf16e412ba as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/b9477331bb3d421dbea23ecf16e412ba 2024-11-13T05:40:06,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/b9477331bb3d421dbea23ecf16e412ba, entries=250, sequenceid=472, filesize=16.8 K 2024-11-13T05:40:06,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/24f065da8a714c36992e7a5f00bffa46 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/24f065da8a714c36992e7a5f00bffa46 2024-11-13T05:40:06,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/24f065da8a714c36992e7a5f00bffa46, entries=150, sequenceid=472, filesize=12.0 K 2024-11-13T05:40:06,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/a542fa2a1ebe4f748e888a25f7124944 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/a542fa2a1ebe4f748e888a25f7124944 2024-11-13T05:40:06,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T05:40:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:06,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:06,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/a542fa2a1ebe4f748e888a25f7124944, entries=150, sequenceid=472, filesize=12.0 K 2024-11-13T05:40:06,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 900ms, sequenceid=472, compaction requested=true 2024-11-13T05:40:06,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:06,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:06,898 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:06,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:06,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:06,898 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:06,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:06,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:06,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:06,900 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:06,900 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/A is initiating minor compaction (all files) 2024-11-13T05:40:06,900 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/A in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:06,900 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/313eeae9c0ec42d09d6606c8b9a5bbe7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/b05196bfb0114172aad931ae62621e45, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/b9477331bb3d421dbea23ecf16e412ba] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=41.8 K 2024-11-13T05:40:06,900 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:06,900 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/B is initiating minor compaction (all files) 2024-11-13T05:40:06,900 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/B in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:06,901 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/66173a37292c473b989fcb48318fb235, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/e541eab3468f4cc69e184e4f7652f431, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/24f065da8a714c36992e7a5f00bffa46] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=37.0 K 2024-11-13T05:40:06,901 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 313eeae9c0ec42d09d6606c8b9a5bbe7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731476403426 2024-11-13T05:40:06,901 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 66173a37292c473b989fcb48318fb235, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731476403426 2024-11-13T05:40:06,902 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting b05196bfb0114172aad931ae62621e45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731476404136 2024-11-13T05:40:06,902 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting e541eab3468f4cc69e184e4f7652f431, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731476404136 2024-11-13T05:40:06,902 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9477331bb3d421dbea23ecf16e412ba, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731476404837 2024-11-13T05:40:06,902 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 24f065da8a714c36992e7a5f00bffa46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731476404837 2024-11-13T05:40:06,927 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:06,928 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/f3bc8f99ccca4d64855f02e610bc9bdf is 50, key is test_row_0/A:col10/1731476405997/Put/seqid=0 2024-11-13T05:40:06,941 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#B#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:06,941 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/f764e14bc9034eef8a62c70cf3072582 is 50, key is test_row_0/B:col10/1731476405997/Put/seqid=0 2024-11-13T05:40:06,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741936_1112 (size=13425) 2024-11-13T05:40:06,968 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/f3bc8f99ccca4d64855f02e610bc9bdf as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/f3bc8f99ccca4d64855f02e610bc9bdf 2024-11-13T05:40:06,976 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/A of 6abd824d4b7a94f2d19ab3e39e94ccfe into f3bc8f99ccca4d64855f02e610bc9bdf(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:06,976 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:06,977 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/A, priority=13, startTime=1731476406898; duration=0sec 2024-11-13T05:40:06,977 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:06,977 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:A 2024-11-13T05:40:06,977 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:06,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741937_1113 (size=13425) 2024-11-13T05:40:06,979 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:06,979 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/C is initiating minor compaction (all files) 2024-11-13T05:40:06,979 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/C in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:06,979 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/a5b78118d84a4a6994789ea4f7bf0542, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/1b09a51fd34e49409e25a7304673bb01, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/a542fa2a1ebe4f748e888a25f7124944] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=37.0 K 2024-11-13T05:40:06,983 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5b78118d84a4a6994789ea4f7bf0542, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731476403426 2024-11-13T05:40:06,987 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b09a51fd34e49409e25a7304673bb01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731476404136 2024-11-13T05:40:06,991 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting a542fa2a1ebe4f748e888a25f7124944, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731476404837 2024-11-13T05:40:06,997 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/f764e14bc9034eef8a62c70cf3072582 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f764e14bc9034eef8a62c70cf3072582 2024-11-13T05:40:07,011 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/B of 6abd824d4b7a94f2d19ab3e39e94ccfe into f764e14bc9034eef8a62c70cf3072582(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:07,017 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:07,017 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/B, priority=13, startTime=1731476406898; duration=0sec 2024-11-13T05:40:07,017 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:07,017 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:B 2024-11-13T05:40:07,018 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#C#compaction#101 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:07,019 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/2998182a239148d9ad7d115401f43669 is 50, key is test_row_0/C:col10/1731476405997/Put/seqid=0 2024-11-13T05:40:07,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741938_1114 (size=13425) 2024-11-13T05:40:07,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T05:40:07,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:07,041 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-13T05:40:07,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/2537081edb6b47f4a04d57da026767b4 is 50, key is test_row_0/A:col10/1731476406024/Put/seqid=0 2024-11-13T05:40:07,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741939_1115 (size=12301) 2024-11-13T05:40:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:40:07,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. as already flushing 2024-11-13T05:40:07,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476467196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:07,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476467196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:07,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476467202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:07,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T05:40:07,285 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-13T05:40:07,285 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:07,285 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-13T05:40:07,285 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:07,286 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-13T05:40:07,287 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:07,287 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-13T05:40:07,287 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:07,290 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:07,290 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:07,290 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:07,290 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:07,291 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:07,291 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:07,291 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:07,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:07,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:07,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:07,291 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:07,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:07,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476467306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:07,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:07,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:07,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476467317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476467316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:07,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:07,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:07,439 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/2998182a239148d9ad7d115401f43669 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/2998182a239148d9ad7d115401f43669 2024-11-13T05:40:07,448 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/C of 6abd824d4b7a94f2d19ab3e39e94ccfe into 2998182a239148d9ad7d115401f43669(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:07,448 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:07,448 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/C, priority=13, startTime=1731476406899; duration=0sec 2024-11-13T05:40:07,448 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:07,448 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:C 2024-11-13T05:40:07,460 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/2537081edb6b47f4a04d57da026767b4 2024-11-13T05:40:07,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/db137f0f301b4fb492c426d62821de58 is 50, key is test_row_0/B:col10/1731476406024/Put/seqid=0 2024-11-13T05:40:07,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741940_1116 (size=12301) 2024-11-13T05:40:07,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:07,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476467517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:07,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:07,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:07,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476467526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476467526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:07,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42880 deadline: 1731476467827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:07,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:07,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42908 deadline: 1731476467836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731476467836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:07,876 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/db137f0f301b4fb492c426d62821de58 2024-11-13T05:40:07,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/ff577814e692404995015cb96c6910cf is 50, key is test_row_0/C:col10/1731476406024/Put/seqid=0 2024-11-13T05:40:07,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741941_1117 (size=12301) 2024-11-13T05:40:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T05:40:08,289 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/ff577814e692404995015cb96c6910cf 2024-11-13T05:40:08,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/2537081edb6b47f4a04d57da026767b4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/2537081edb6b47f4a04d57da026767b4 2024-11-13T05:40:08,300 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/2537081edb6b47f4a04d57da026767b4, entries=150, sequenceid=493, filesize=12.0 K 2024-11-13T05:40:08,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/db137f0f301b4fb492c426d62821de58 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/db137f0f301b4fb492c426d62821de58 2024-11-13T05:40:08,307 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/db137f0f301b4fb492c426d62821de58, entries=150, sequenceid=493, filesize=12.0 K 2024-11-13T05:40:08,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/ff577814e692404995015cb96c6910cf as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/ff577814e692404995015cb96c6910cf 2024-11-13T05:40:08,317 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/ff577814e692404995015cb96c6910cf, entries=150, sequenceid=493, filesize=12.0 K 2024-11-13T05:40:08,318 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 1277ms, sequenceid=493, compaction requested=false 2024-11-13T05:40:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-13T05:40:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-13T05:40:08,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-13T05:40:08,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2040 sec 2024-11-13T05:40:08,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.2130 sec 2024-11-13T05:40:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:40:08,343 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:08,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T05:40:08,345 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-13T05:40:08,345 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:08,345 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:08,345 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:08,345 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:08,348 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-13T05:40:08,348 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:08,349 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:08,349 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:08,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:08,349 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-13T05:40:08,349 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:08,349 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:08,349 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:08,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:08,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/0d1184f70d9e40e2b0e1b1624e4dd6ec is 50, key is test_row_0/A:col10/1731476407200/Put/seqid=0 2024-11-13T05:40:08,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741942_1118 (size=12301) 2024-11-13T05:40:08,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/0d1184f70d9e40e2b0e1b1624e4dd6ec 2024-11-13T05:40:08,391 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:40:08,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/ae49ff307c91475c88f32abcb2229fac is 50, key is test_row_0/B:col10/1731476407200/Put/seqid=0 2024-11-13T05:40:08,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741943_1119 (size=12301) 2024-11-13T05:40:08,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/ae49ff307c91475c88f32abcb2229fac 2024-11-13T05:40:08,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/68146bd423324b129915f9d7618a5f39 is 50, key is test_row_0/C:col10/1731476407200/Put/seqid=0 2024-11-13T05:40:08,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741944_1120 (size=12301) 2024-11-13T05:40:09,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/68146bd423324b129915f9d7618a5f39 2024-11-13T05:40:09,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/0d1184f70d9e40e2b0e1b1624e4dd6ec as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0d1184f70d9e40e2b0e1b1624e4dd6ec 2024-11-13T05:40:09,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0d1184f70d9e40e2b0e1b1624e4dd6ec, entries=150, sequenceid=512, filesize=12.0 K 2024-11-13T05:40:09,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/ae49ff307c91475c88f32abcb2229fac as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/ae49ff307c91475c88f32abcb2229fac 2024-11-13T05:40:09,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/ae49ff307c91475c88f32abcb2229fac, entries=150, sequenceid=512, filesize=12.0 K 2024-11-13T05:40:09,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/68146bd423324b129915f9d7618a5f39 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/68146bd423324b129915f9d7618a5f39 2024-11-13T05:40:09,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/68146bd423324b129915f9d7618a5f39, entries=150, sequenceid=512, filesize=12.0 K 2024-11-13T05:40:09,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=13.42 KB/13740 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 985ms, sequenceid=512, compaction requested=true 2024-11-13T05:40:09,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:09,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:09,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:09,329 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:09,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:09,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:09,329 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:09,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6abd824d4b7a94f2d19ab3e39e94ccfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:09,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:09,330 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:09,330 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:09,330 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/A is initiating minor compaction (all files) 2024-11-13T05:40:09,330 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/B is initiating minor compaction (all files) 2024-11-13T05:40:09,330 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/B in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:09,330 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/A in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:09,330 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/f3bc8f99ccca4d64855f02e610bc9bdf, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/2537081edb6b47f4a04d57da026767b4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0d1184f70d9e40e2b0e1b1624e4dd6ec] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=37.1 K 2024-11-13T05:40:09,330 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f764e14bc9034eef8a62c70cf3072582, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/db137f0f301b4fb492c426d62821de58, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/ae49ff307c91475c88f32abcb2229fac] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=37.1 K 2024-11-13T05:40:09,331 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3bc8f99ccca4d64855f02e610bc9bdf, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731476404837 2024-11-13T05:40:09,331 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting f764e14bc9034eef8a62c70cf3072582, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731476404837 2024-11-13T05:40:09,331 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2537081edb6b47f4a04d57da026767b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731476406014 2024-11-13T05:40:09,331 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting db137f0f301b4fb492c426d62821de58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731476406014 2024-11-13T05:40:09,332 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting ae49ff307c91475c88f32abcb2229fac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731476407189 2024-11-13T05:40:09,332 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d1184f70d9e40e2b0e1b1624e4dd6ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731476407189 2024-11-13T05:40:09,345 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#A#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:09,346 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/f7b2b2a1cc28413aaab70f3c0927180c is 50, key is test_row_0/A:col10/1731476407200/Put/seqid=0 2024-11-13T05:40:09,348 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#B#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:09,349 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/0ae059ef836e4f35b2375ba882f6d54a is 50, key is test_row_0/B:col10/1731476407200/Put/seqid=0 2024-11-13T05:40:09,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741945_1121 (size=13527) 2024-11-13T05:40:09,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741946_1122 (size=13527) 2024-11-13T05:40:09,759 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/f7b2b2a1cc28413aaab70f3c0927180c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/f7b2b2a1cc28413aaab70f3c0927180c 2024-11-13T05:40:09,759 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/0ae059ef836e4f35b2375ba882f6d54a as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0ae059ef836e4f35b2375ba882f6d54a 2024-11-13T05:40:09,765 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/A of 6abd824d4b7a94f2d19ab3e39e94ccfe into f7b2b2a1cc28413aaab70f3c0927180c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:09,765 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/B of 6abd824d4b7a94f2d19ab3e39e94ccfe into 0ae059ef836e4f35b2375ba882f6d54a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:09,765 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:09,765 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:09,765 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/A, priority=13, startTime=1731476409329; duration=0sec 2024-11-13T05:40:09,765 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/B, priority=13, startTime=1731476409329; duration=0sec 2024-11-13T05:40:09,765 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:09,765 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:B 2024-11-13T05:40:09,765 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:09,765 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:A 2024-11-13T05:40:09,765 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:09,766 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:09,766 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 6abd824d4b7a94f2d19ab3e39e94ccfe/C is initiating minor compaction (all files) 2024-11-13T05:40:09,766 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6abd824d4b7a94f2d19ab3e39e94ccfe/C in TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:09,766 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/2998182a239148d9ad7d115401f43669, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/ff577814e692404995015cb96c6910cf, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/68146bd423324b129915f9d7618a5f39] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp, totalSize=37.1 K 2024-11-13T05:40:09,767 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 2998182a239148d9ad7d115401f43669, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731476404837 2024-11-13T05:40:09,767 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting ff577814e692404995015cb96c6910cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731476406014 2024-11-13T05:40:09,768 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 68146bd423324b129915f9d7618a5f39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731476407189 2024-11-13T05:40:09,776 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6abd824d4b7a94f2d19ab3e39e94ccfe#C#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:09,776 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/c2f5d1482b42418690a7e3e9ea635a60 is 50, key is test_row_0/C:col10/1731476407200/Put/seqid=0 2024-11-13T05:40:09,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741947_1123 (size=13527) 2024-11-13T05:40:10,186 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/c2f5d1482b42418690a7e3e9ea635a60 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/c2f5d1482b42418690a7e3e9ea635a60 2024-11-13T05:40:10,192 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6abd824d4b7a94f2d19ab3e39e94ccfe/C of 6abd824d4b7a94f2d19ab3e39e94ccfe into c2f5d1482b42418690a7e3e9ea635a60(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:10,192 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: 2024-11-13T05:40:10,192 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., storeName=6abd824d4b7a94f2d19ab3e39e94ccfe/C, priority=13, startTime=1731476409329; duration=0sec 2024-11-13T05:40:10,193 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:10,193 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6abd824d4b7a94f2d19ab3e39e94ccfe:C 2024-11-13T05:40:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T05:40:10,286 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:14,361 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-13T05:40:14,361 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:14,361 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:14,362 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:14,362 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:14,411 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-13T05:40:14,411 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:14,411 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:14,412 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:14,412 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:14,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T05:40:14,412 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-13T05:40:14,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-13T05:40:14,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-13T05:40:14,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-13T05:40:14,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-13T05:40:14,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T05:40:14,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2863 2024-11-13T05:40:14,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2908 2024-11-13T05:40:14,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T05:40:14,413 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1361 2024-11-13T05:40:14,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4082 rows 2024-11-13T05:40:14,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1361 2024-11-13T05:40:14,414 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4079 rows 2024-11-13T05:40:14,414 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:40:14,414 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:40:14,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:14,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:14,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:14,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T05:40:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T05:40:14,427 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476414427"}]},"ts":"1731476414427"} 2024-11-13T05:40:14,429 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T05:40:14,429 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T05:40:14,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:40:14,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6abd824d4b7a94f2d19ab3e39e94ccfe, UNASSIGN}] 2024-11-13T05:40:14,436 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6abd824d4b7a94f2d19ab3e39e94ccfe, UNASSIGN 2024-11-13T05:40:14,437 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=6abd824d4b7a94f2d19ab3e39e94ccfe, regionState=CLOSING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:14,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6abd824d4b7a94f2d19ab3e39e94ccfe, UNASSIGN because future has completed 2024-11-13T05:40:14,440 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:40:14,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:40:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T05:40:14,598 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(122): Close 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:40:14,598 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:40:14,599 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1722): Closing 6abd824d4b7a94f2d19ab3e39e94ccfe, disabling compactions & flushes 2024-11-13T05:40:14,599 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:14,599 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:14,599 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. after waiting 0 ms 2024-11-13T05:40:14,599 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:14,599 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(2902): Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-13T05:40:14,604 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/658d1d1c9a8a42ad8b8ac36017a08d77 is 50, key is test_row_0/A:col10/1731476408346/Put/seqid=0 2024-11-13T05:40:14,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741948_1124 (size=12301) 2024-11-13T05:40:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T05:40:15,009 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/658d1d1c9a8a42ad8b8ac36017a08d77 2024-11-13T05:40:15,017 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/399c09b3e7a24c8298007d5c666173d7 is 50, key is test_row_0/B:col10/1731476408346/Put/seqid=0 2024-11-13T05:40:15,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741949_1125 (size=12301) 2024-11-13T05:40:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T05:40:15,423 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/399c09b3e7a24c8298007d5c666173d7 2024-11-13T05:40:15,438 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/3c4024716ee549e2a90e112ab2e027fa is 50, key is test_row_0/C:col10/1731476408346/Put/seqid=0 2024-11-13T05:40:15,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741950_1126 (size=12301) 2024-11-13T05:40:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T05:40:15,843 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/3c4024716ee549e2a90e112ab2e027fa 2024-11-13T05:40:15,852 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/A/658d1d1c9a8a42ad8b8ac36017a08d77 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/658d1d1c9a8a42ad8b8ac36017a08d77 2024-11-13T05:40:15,858 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/658d1d1c9a8a42ad8b8ac36017a08d77, entries=150, sequenceid=522, filesize=12.0 K 2024-11-13T05:40:15,859 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/B/399c09b3e7a24c8298007d5c666173d7 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/399c09b3e7a24c8298007d5c666173d7 2024-11-13T05:40:15,864 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/399c09b3e7a24c8298007d5c666173d7, entries=150, sequenceid=522, filesize=12.0 K 2024-11-13T05:40:15,865 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/.tmp/C/3c4024716ee549e2a90e112ab2e027fa as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/3c4024716ee549e2a90e112ab2e027fa 2024-11-13T05:40:15,870 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/3c4024716ee549e2a90e112ab2e027fa, entries=150, sequenceid=522, filesize=12.0 K 2024-11-13T05:40:15,871 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 1271ms, sequenceid=522, compaction requested=false 2024-11-13T05:40:15,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/fca911b09800409b918997af7426f2ad, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/e3472adb5d1142519f0b8b4df68e07e5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/2523210a3045447da24ac8566b91c633, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ff8facb8052b41da8b7645aee30a4d68, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/32af0583b0ce4b7aafdb1d142f319d89, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/366e161d49224680b69f3d93f38de9fb, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8f1636ff445449b38c52d452a437fd14, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0d98e4e53e1044c48bb6fa5d8bbb83b9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/73d0ca667cd7496aa5d95676726513d2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8a7351059e4741138fcdf6496231a3df, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/163896c8328140d780d45295b7eb17e9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/6cf05076cfa54190a8fe43ad192bde7e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/a09b4bb1f63a4e898ffae7803f05d433, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/14f9631a69ab4c03b7bdc91407218cdb, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0ce75d7010db4f0192676d211c2f46fe, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ec92832c4e9c4404ae614285a7f31d5f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/7f9f5cdb145044d4ae9275b0749eb6c0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5bd931cfa250485fb861377f24d9a918, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5f893eee04fb424ca62d15b8a2038564, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ce554876c6e84fe38e13b7b6eb6bcbae, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/841290a47247422d856a67be1a46e3dc, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/13f1b0021cb2486d9e47a247133fba6b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5cab3b4fff9b467898972204168f84a4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8edf8c0e9c92477c8983792aa2e1b6c2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/4ea6a5aa6d8445a2b8defc8585224659, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/a13a026f038e4a4a826087ce4d989b30, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/13681015645140c1a26ff201c74dd7ea, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/12562f6d50bc4430abf2e44a057d1451, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/943120eaeba9425a8525f31d86b4cde5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/313eeae9c0ec42d09d6606c8b9a5bbe7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/911fc67f089b45c9ae8a65ec30b321f6, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/b05196bfb0114172aad931ae62621e45, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/b9477331bb3d421dbea23ecf16e412ba, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/f3bc8f99ccca4d64855f02e610bc9bdf, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/2537081edb6b47f4a04d57da026767b4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0d1184f70d9e40e2b0e1b1624e4dd6ec] to archive 2024-11-13T05:40:15,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:40:15,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/fca911b09800409b918997af7426f2ad to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/fca911b09800409b918997af7426f2ad 2024-11-13T05:40:15,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/e3472adb5d1142519f0b8b4df68e07e5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/e3472adb5d1142519f0b8b4df68e07e5 2024-11-13T05:40:15,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/2523210a3045447da24ac8566b91c633 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/2523210a3045447da24ac8566b91c633 2024-11-13T05:40:15,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ff8facb8052b41da8b7645aee30a4d68 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ff8facb8052b41da8b7645aee30a4d68 2024-11-13T05:40:15,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/32af0583b0ce4b7aafdb1d142f319d89 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/32af0583b0ce4b7aafdb1d142f319d89 2024-11-13T05:40:15,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/366e161d49224680b69f3d93f38de9fb to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/366e161d49224680b69f3d93f38de9fb 2024-11-13T05:40:15,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8f1636ff445449b38c52d452a437fd14 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8f1636ff445449b38c52d452a437fd14 2024-11-13T05:40:15,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0d98e4e53e1044c48bb6fa5d8bbb83b9 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0d98e4e53e1044c48bb6fa5d8bbb83b9 2024-11-13T05:40:15,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/73d0ca667cd7496aa5d95676726513d2 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/73d0ca667cd7496aa5d95676726513d2 2024-11-13T05:40:15,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8a7351059e4741138fcdf6496231a3df to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8a7351059e4741138fcdf6496231a3df 2024-11-13T05:40:15,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/163896c8328140d780d45295b7eb17e9 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/163896c8328140d780d45295b7eb17e9 2024-11-13T05:40:15,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/6cf05076cfa54190a8fe43ad192bde7e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/6cf05076cfa54190a8fe43ad192bde7e 2024-11-13T05:40:15,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/a09b4bb1f63a4e898ffae7803f05d433 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/a09b4bb1f63a4e898ffae7803f05d433 2024-11-13T05:40:15,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/14f9631a69ab4c03b7bdc91407218cdb to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/14f9631a69ab4c03b7bdc91407218cdb 2024-11-13T05:40:15,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0ce75d7010db4f0192676d211c2f46fe to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0ce75d7010db4f0192676d211c2f46fe 2024-11-13T05:40:15,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ec92832c4e9c4404ae614285a7f31d5f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ec92832c4e9c4404ae614285a7f31d5f 2024-11-13T05:40:15,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/7f9f5cdb145044d4ae9275b0749eb6c0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/7f9f5cdb145044d4ae9275b0749eb6c0 2024-11-13T05:40:15,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5bd931cfa250485fb861377f24d9a918 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5bd931cfa250485fb861377f24d9a918 2024-11-13T05:40:15,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5f893eee04fb424ca62d15b8a2038564 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5f893eee04fb424ca62d15b8a2038564 2024-11-13T05:40:15,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ce554876c6e84fe38e13b7b6eb6bcbae to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/ce554876c6e84fe38e13b7b6eb6bcbae 2024-11-13T05:40:15,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/841290a47247422d856a67be1a46e3dc to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/841290a47247422d856a67be1a46e3dc 2024-11-13T05:40:15,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/13f1b0021cb2486d9e47a247133fba6b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/13f1b0021cb2486d9e47a247133fba6b 2024-11-13T05:40:15,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5cab3b4fff9b467898972204168f84a4 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/5cab3b4fff9b467898972204168f84a4 2024-11-13T05:40:15,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8edf8c0e9c92477c8983792aa2e1b6c2 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/8edf8c0e9c92477c8983792aa2e1b6c2 2024-11-13T05:40:15,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/4ea6a5aa6d8445a2b8defc8585224659 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/4ea6a5aa6d8445a2b8defc8585224659 2024-11-13T05:40:15,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/a13a026f038e4a4a826087ce4d989b30 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/a13a026f038e4a4a826087ce4d989b30 2024-11-13T05:40:15,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/13681015645140c1a26ff201c74dd7ea to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/13681015645140c1a26ff201c74dd7ea 2024-11-13T05:40:15,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/12562f6d50bc4430abf2e44a057d1451 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/12562f6d50bc4430abf2e44a057d1451 2024-11-13T05:40:15,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/943120eaeba9425a8525f31d86b4cde5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/943120eaeba9425a8525f31d86b4cde5 2024-11-13T05:40:15,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/313eeae9c0ec42d09d6606c8b9a5bbe7 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/313eeae9c0ec42d09d6606c8b9a5bbe7 2024-11-13T05:40:15,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/911fc67f089b45c9ae8a65ec30b321f6 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/911fc67f089b45c9ae8a65ec30b321f6 2024-11-13T05:40:15,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/b05196bfb0114172aad931ae62621e45 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/b05196bfb0114172aad931ae62621e45 2024-11-13T05:40:15,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/b9477331bb3d421dbea23ecf16e412ba to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/b9477331bb3d421dbea23ecf16e412ba 2024-11-13T05:40:15,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/f3bc8f99ccca4d64855f02e610bc9bdf to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/f3bc8f99ccca4d64855f02e610bc9bdf 2024-11-13T05:40:15,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/2537081edb6b47f4a04d57da026767b4 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/2537081edb6b47f4a04d57da026767b4 2024-11-13T05:40:15,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0d1184f70d9e40e2b0e1b1624e4dd6ec to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/0d1184f70d9e40e2b0e1b1624e4dd6ec 2024-11-13T05:40:15,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/c97f6407a97a445098ac1fa9823ef712, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/097b42762fa841b88d54145422962970, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1448118c03704b66a8a695d5b5885bfa, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/952b4e5d19cb4a968af9e0a3600c712c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/6c05da63314c4191a8a7e62d11d59caf, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/4ed3bd516c744fd59a9be270924bb512, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/4a824387ff4742abaddb757aa5aeef51, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f22c4c5c5cae4b87a536765ed56a54f3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f22979d52a414451b2804b0a468cdf47, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/5a84fb8e8cef4e04b44e56ceaebb9b6b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/abfdcc7ea82c4d3c8f2ac144967821dc, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/eb0276bb8b814f91a020b89f42819e4e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/641cdeb4bf614571822695b27cbe513b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/ca76f3b8c5ad4a52ad78a3e665592639, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/674230cd601042d5a4661765f47235e7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1c12e7ae8671427d849746ea5729a4a2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1f1c05158cae4416860497ce4d1ecf7b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/7985a8244c5145809993c562e36c4111, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/b11fbe021c544a8f8b4766b6dcfbfa38, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/302e7c58657a4b2795a3671f7979d3f7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0a1aed12ba324bcbb78f7569f5130310, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/bda5ba4388f240a282cff85b76e43032, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/6db953a90dcb47cf9a114797c8284ea4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/507c260bfac143aeae083ebd009b7de8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/b627f2e4984e4905b94214c83699fdc6, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f7478b406efb4c77bda53cde272773f6, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/eb7b81d812094c63a61cdbabdcb07f54, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/053dd5c872474a44bc93ec1603a031fd, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0d27e79389214b80a6fa858d1f8a54bc, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/66173a37292c473b989fcb48318fb235, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/9e00f6f28d8d40498c8c3861e746c0c9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/e541eab3468f4cc69e184e4f7652f431, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f764e14bc9034eef8a62c70cf3072582, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/24f065da8a714c36992e7a5f00bffa46, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/db137f0f301b4fb492c426d62821de58, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/ae49ff307c91475c88f32abcb2229fac] to archive 2024-11-13T05:40:15,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:40:15,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/c97f6407a97a445098ac1fa9823ef712 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/c97f6407a97a445098ac1fa9823ef712 2024-11-13T05:40:15,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/097b42762fa841b88d54145422962970 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/097b42762fa841b88d54145422962970 2024-11-13T05:40:15,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1448118c03704b66a8a695d5b5885bfa to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1448118c03704b66a8a695d5b5885bfa 2024-11-13T05:40:15,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/952b4e5d19cb4a968af9e0a3600c712c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/952b4e5d19cb4a968af9e0a3600c712c 2024-11-13T05:40:15,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/6c05da63314c4191a8a7e62d11d59caf to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/6c05da63314c4191a8a7e62d11d59caf 2024-11-13T05:40:15,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/4ed3bd516c744fd59a9be270924bb512 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/4ed3bd516c744fd59a9be270924bb512 2024-11-13T05:40:15,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/4a824387ff4742abaddb757aa5aeef51 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/4a824387ff4742abaddb757aa5aeef51 2024-11-13T05:40:15,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f22c4c5c5cae4b87a536765ed56a54f3 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f22c4c5c5cae4b87a536765ed56a54f3 2024-11-13T05:40:15,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f22979d52a414451b2804b0a468cdf47 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f22979d52a414451b2804b0a468cdf47 2024-11-13T05:40:15,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/5a84fb8e8cef4e04b44e56ceaebb9b6b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/5a84fb8e8cef4e04b44e56ceaebb9b6b 2024-11-13T05:40:15,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/abfdcc7ea82c4d3c8f2ac144967821dc to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/abfdcc7ea82c4d3c8f2ac144967821dc 2024-11-13T05:40:15,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/eb0276bb8b814f91a020b89f42819e4e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/eb0276bb8b814f91a020b89f42819e4e 2024-11-13T05:40:15,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/641cdeb4bf614571822695b27cbe513b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/641cdeb4bf614571822695b27cbe513b 2024-11-13T05:40:15,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/ca76f3b8c5ad4a52ad78a3e665592639 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/ca76f3b8c5ad4a52ad78a3e665592639 2024-11-13T05:40:15,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/674230cd601042d5a4661765f47235e7 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/674230cd601042d5a4661765f47235e7 2024-11-13T05:40:15,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1c12e7ae8671427d849746ea5729a4a2 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1c12e7ae8671427d849746ea5729a4a2 2024-11-13T05:40:15,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1f1c05158cae4416860497ce4d1ecf7b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/1f1c05158cae4416860497ce4d1ecf7b 2024-11-13T05:40:15,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/7985a8244c5145809993c562e36c4111 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/7985a8244c5145809993c562e36c4111 2024-11-13T05:40:15,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/b11fbe021c544a8f8b4766b6dcfbfa38 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/b11fbe021c544a8f8b4766b6dcfbfa38 2024-11-13T05:40:15,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/302e7c58657a4b2795a3671f7979d3f7 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/302e7c58657a4b2795a3671f7979d3f7 2024-11-13T05:40:15,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0a1aed12ba324bcbb78f7569f5130310 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0a1aed12ba324bcbb78f7569f5130310 2024-11-13T05:40:15,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/bda5ba4388f240a282cff85b76e43032 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/bda5ba4388f240a282cff85b76e43032 2024-11-13T05:40:15,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/6db953a90dcb47cf9a114797c8284ea4 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/6db953a90dcb47cf9a114797c8284ea4 2024-11-13T05:40:15,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/507c260bfac143aeae083ebd009b7de8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/507c260bfac143aeae083ebd009b7de8 2024-11-13T05:40:15,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/b627f2e4984e4905b94214c83699fdc6 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/b627f2e4984e4905b94214c83699fdc6 2024-11-13T05:40:15,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f7478b406efb4c77bda53cde272773f6 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f7478b406efb4c77bda53cde272773f6 2024-11-13T05:40:15,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/eb7b81d812094c63a61cdbabdcb07f54 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/eb7b81d812094c63a61cdbabdcb07f54 2024-11-13T05:40:15,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/053dd5c872474a44bc93ec1603a031fd to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/053dd5c872474a44bc93ec1603a031fd 2024-11-13T05:40:15,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0d27e79389214b80a6fa858d1f8a54bc to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0d27e79389214b80a6fa858d1f8a54bc 2024-11-13T05:40:15,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/66173a37292c473b989fcb48318fb235 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/66173a37292c473b989fcb48318fb235 2024-11-13T05:40:15,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/9e00f6f28d8d40498c8c3861e746c0c9 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/9e00f6f28d8d40498c8c3861e746c0c9 2024-11-13T05:40:15,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/e541eab3468f4cc69e184e4f7652f431 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/e541eab3468f4cc69e184e4f7652f431 2024-11-13T05:40:15,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f764e14bc9034eef8a62c70cf3072582 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/f764e14bc9034eef8a62c70cf3072582 2024-11-13T05:40:15,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/24f065da8a714c36992e7a5f00bffa46 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/24f065da8a714c36992e7a5f00bffa46 2024-11-13T05:40:15,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/db137f0f301b4fb492c426d62821de58 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/db137f0f301b4fb492c426d62821de58 2024-11-13T05:40:15,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/ae49ff307c91475c88f32abcb2229fac to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/ae49ff307c91475c88f32abcb2229fac 2024-11-13T05:40:15,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5e4b14f0baf143509d6bef4cbf220292, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/01f87f6d5ad14092b4cc353b09ed8920, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/9b19a3df54cc40b6a766a45d717d7efb, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/e13970f9ba76452f9c0086296986910d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/d3722b269be14cc28f54864fc3d50a2f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/137fbaad12c04d9eab7bb0885673ac2d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/b4d4568570794c47b9a2b2540b017200, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/ddba5ab719964ab1ba9de7c1dab9e1bb, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/59477aff38c941818a138be7b675df89, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5a370675bbbf44af821482dbfb97cafe, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/d5742a3a530346d7b98d7b7caf08f076, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5ae150d869504c53a9bc8406491af2f1, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/98c44411a3744de0bffb83ba28fcb4b3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/4e16b06384844b1293e412addb9f335e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5ed3142deb404c5d9430d7e06df7e815, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/c3c84ebdae074927935dfd0f3c83ff3e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/faeddd80e7db4b5d9279c56f79e8b967, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/945d45fc464d48d2ba913fe2f8c34cd8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/38ad84b2e2784f52abbe497de242c6ff, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/8c2902a6b58f424eb509dd51cbfdca73, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5cc69261809b4c97a7f1cd62dd0aa3dc, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/34ffaefbc14a4bc8ac4eda1f66d7a833, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/9dab42c13fb34a2f9304bb1a69cdf34e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/fd49f4bc3837435c8dc6c2a28251c265, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/abf81e11856f4306a0f8cfc6a2672c02, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/c99d446553424c5190bd2c8b96d25ba2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/3030c7a9c4c243c0be23632cc94e8f4f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/e89af198f73f44b5a3fe9a65d472dbc5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/62c5d890c8a54198a77e3359c67ea8b0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/a5b78118d84a4a6994789ea4f7bf0542, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/636673c08c57483383c6b9cad82bc91e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/1b09a51fd34e49409e25a7304673bb01, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/2998182a239148d9ad7d115401f43669, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/a542fa2a1ebe4f748e888a25f7124944, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/ff577814e692404995015cb96c6910cf, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/68146bd423324b129915f9d7618a5f39] to archive 2024-11-13T05:40:15,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:40:15,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5e4b14f0baf143509d6bef4cbf220292 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5e4b14f0baf143509d6bef4cbf220292 2024-11-13T05:40:16,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/01f87f6d5ad14092b4cc353b09ed8920 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/01f87f6d5ad14092b4cc353b09ed8920 2024-11-13T05:40:16,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/9b19a3df54cc40b6a766a45d717d7efb to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/9b19a3df54cc40b6a766a45d717d7efb 2024-11-13T05:40:16,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/e13970f9ba76452f9c0086296986910d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/e13970f9ba76452f9c0086296986910d 2024-11-13T05:40:16,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/d3722b269be14cc28f54864fc3d50a2f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/d3722b269be14cc28f54864fc3d50a2f 2024-11-13T05:40:16,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/137fbaad12c04d9eab7bb0885673ac2d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/137fbaad12c04d9eab7bb0885673ac2d 2024-11-13T05:40:16,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/b4d4568570794c47b9a2b2540b017200 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/b4d4568570794c47b9a2b2540b017200 2024-11-13T05:40:16,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/ddba5ab719964ab1ba9de7c1dab9e1bb to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/ddba5ab719964ab1ba9de7c1dab9e1bb 2024-11-13T05:40:16,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/59477aff38c941818a138be7b675df89 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/59477aff38c941818a138be7b675df89 2024-11-13T05:40:16,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5a370675bbbf44af821482dbfb97cafe to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5a370675bbbf44af821482dbfb97cafe 2024-11-13T05:40:16,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/d5742a3a530346d7b98d7b7caf08f076 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/d5742a3a530346d7b98d7b7caf08f076 2024-11-13T05:40:16,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5ae150d869504c53a9bc8406491af2f1 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5ae150d869504c53a9bc8406491af2f1 2024-11-13T05:40:16,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/98c44411a3744de0bffb83ba28fcb4b3 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/98c44411a3744de0bffb83ba28fcb4b3 2024-11-13T05:40:16,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/4e16b06384844b1293e412addb9f335e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/4e16b06384844b1293e412addb9f335e 2024-11-13T05:40:16,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5ed3142deb404c5d9430d7e06df7e815 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5ed3142deb404c5d9430d7e06df7e815 2024-11-13T05:40:16,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/c3c84ebdae074927935dfd0f3c83ff3e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/c3c84ebdae074927935dfd0f3c83ff3e 2024-11-13T05:40:16,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/faeddd80e7db4b5d9279c56f79e8b967 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/faeddd80e7db4b5d9279c56f79e8b967 2024-11-13T05:40:16,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/945d45fc464d48d2ba913fe2f8c34cd8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/945d45fc464d48d2ba913fe2f8c34cd8 2024-11-13T05:40:16,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/38ad84b2e2784f52abbe497de242c6ff to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/38ad84b2e2784f52abbe497de242c6ff 2024-11-13T05:40:16,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/8c2902a6b58f424eb509dd51cbfdca73 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/8c2902a6b58f424eb509dd51cbfdca73 2024-11-13T05:40:16,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5cc69261809b4c97a7f1cd62dd0aa3dc to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/5cc69261809b4c97a7f1cd62dd0aa3dc 2024-11-13T05:40:16,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/34ffaefbc14a4bc8ac4eda1f66d7a833 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/34ffaefbc14a4bc8ac4eda1f66d7a833 2024-11-13T05:40:16,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/9dab42c13fb34a2f9304bb1a69cdf34e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/9dab42c13fb34a2f9304bb1a69cdf34e 2024-11-13T05:40:16,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/fd49f4bc3837435c8dc6c2a28251c265 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/fd49f4bc3837435c8dc6c2a28251c265 2024-11-13T05:40:16,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/abf81e11856f4306a0f8cfc6a2672c02 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/abf81e11856f4306a0f8cfc6a2672c02 2024-11-13T05:40:16,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/c99d446553424c5190bd2c8b96d25ba2 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/c99d446553424c5190bd2c8b96d25ba2 2024-11-13T05:40:16,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/3030c7a9c4c243c0be23632cc94e8f4f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/3030c7a9c4c243c0be23632cc94e8f4f 2024-11-13T05:40:16,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/e89af198f73f44b5a3fe9a65d472dbc5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/e89af198f73f44b5a3fe9a65d472dbc5 2024-11-13T05:40:16,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/62c5d890c8a54198a77e3359c67ea8b0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/62c5d890c8a54198a77e3359c67ea8b0 2024-11-13T05:40:16,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/a5b78118d84a4a6994789ea4f7bf0542 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/a5b78118d84a4a6994789ea4f7bf0542 2024-11-13T05:40:16,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/636673c08c57483383c6b9cad82bc91e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/636673c08c57483383c6b9cad82bc91e 2024-11-13T05:40:16,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/1b09a51fd34e49409e25a7304673bb01 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/1b09a51fd34e49409e25a7304673bb01 2024-11-13T05:40:16,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/2998182a239148d9ad7d115401f43669 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/2998182a239148d9ad7d115401f43669 2024-11-13T05:40:16,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/a542fa2a1ebe4f748e888a25f7124944 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/a542fa2a1ebe4f748e888a25f7124944 2024-11-13T05:40:16,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/ff577814e692404995015cb96c6910cf to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/ff577814e692404995015cb96c6910cf 2024-11-13T05:40:16,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/68146bd423324b129915f9d7618a5f39 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/68146bd423324b129915f9d7618a5f39 2024-11-13T05:40:16,055 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/recovered.edits/525.seqid, newMaxSeqId=525, maxSeqId=1 2024-11-13T05:40:16,056 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. 2024-11-13T05:40:16,057 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1676): Region close journal for 6abd824d4b7a94f2d19ab3e39e94ccfe: Waiting for close lock at 1731476414598Running coprocessor pre-close hooks at 1731476414599 (+1 ms)Disabling compacts and flushes for region at 1731476414599Disabling writes for close at 1731476414599Obtaining lock to block concurrent updates at 1731476414599Preparing flush snapshotting stores in 6abd824d4b7a94f2d19ab3e39e94ccfe at 1731476414599Finished memstore snapshotting TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731476414599Flushing stores of TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe. at 1731476414600 (+1 ms)Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe/A: creating writer at 1731476414600Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe/A: appending metadata at 1731476414603 (+3 ms)Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe/A: closing flushed file at 1731476414603Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe/B: creating writer at 1731476415014 (+411 ms)Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe/B: appending metadata at 1731476415017 (+3 ms)Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe/B: closing flushed file at 1731476415017Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe/C: creating writer at 1731476415433 (+416 ms)Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe/C: appending metadata at 1731476415437 (+4 ms)Flushing 6abd824d4b7a94f2d19ab3e39e94ccfe/C: closing flushed file at 1731476415437Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e263f41: reopening flushed file at 1731476415851 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e420fb7: reopening flushed file at 1731476415858 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cae0799: reopening flushed file at 1731476415864 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 6abd824d4b7a94f2d19ab3e39e94ccfe in 1271ms, sequenceid=522, compaction requested=false at 1731476415871 (+7 ms)Writing region close event to WAL at 1731476416051 (+180 ms)Running coprocessor post-close hooks at 1731476416056 (+5 ms)Closed at 1731476416056 2024-11-13T05:40:16,060 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(157): Closed 6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:40:16,060 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=6abd824d4b7a94f2d19ab3e39e94ccfe, regionState=CLOSED 2024-11-13T05:40:16,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:40:16,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-13T05:40:16,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseRegionProcedure 6abd824d4b7a94f2d19ab3e39e94ccfe, server=bc386b7bf2fe,45837,1731476380128 in 1.6240 sec 2024-11-13T05:40:16,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-13T05:40:16,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6abd824d4b7a94f2d19ab3e39e94ccfe, UNASSIGN in 1.6320 sec 2024-11-13T05:40:16,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-13T05:40:16,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6390 sec 2024-11-13T05:40:16,073 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476416073"}]},"ts":"1731476416073"} 2024-11-13T05:40:16,075 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T05:40:16,075 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T05:40:16,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6540 sec 2024-11-13T05:40:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T05:40:16,577 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:16,580 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T05:40:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:16,584 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T05:40:16,587 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=23, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:16,592 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:40:16,596 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/recovered.edits] 2024-11-13T05:40:16,600 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/658d1d1c9a8a42ad8b8ac36017a08d77 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/658d1d1c9a8a42ad8b8ac36017a08d77 2024-11-13T05:40:16,602 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/f7b2b2a1cc28413aaab70f3c0927180c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/A/f7b2b2a1cc28413aaab70f3c0927180c 2024-11-13T05:40:16,605 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0ae059ef836e4f35b2375ba882f6d54a to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/0ae059ef836e4f35b2375ba882f6d54a 2024-11-13T05:40:16,607 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/399c09b3e7a24c8298007d5c666173d7 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/B/399c09b3e7a24c8298007d5c666173d7 2024-11-13T05:40:16,609 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/3c4024716ee549e2a90e112ab2e027fa to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/3c4024716ee549e2a90e112ab2e027fa 2024-11-13T05:40:16,611 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/c2f5d1482b42418690a7e3e9ea635a60 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/C/c2f5d1482b42418690a7e3e9ea635a60 2024-11-13T05:40:16,614 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/recovered.edits/525.seqid to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe/recovered.edits/525.seqid 2024-11-13T05:40:16,614 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/6abd824d4b7a94f2d19ab3e39e94ccfe 2024-11-13T05:40:16,615 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T05:40:16,619 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=23, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:16,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-13T05:40:16,628 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T05:40:16,661 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T05:40:16,663 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=23, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:16,663 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T05:40:16,663 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731476416663"}]},"ts":"9223372036854775807"} 2024-11-13T05:40:16,668 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T05:40:16,668 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6abd824d4b7a94f2d19ab3e39e94ccfe, NAME => 'TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T05:40:16,668 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T05:40:16,669 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731476416668"}]},"ts":"9223372036854775807"} 2024-11-13T05:40:16,672 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T05:40:16,673 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=23, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:16,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 93 msec 2024-11-13T05:40:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T05:40:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T05:40:16,698 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:16,724 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;bc386b7bf2fe:45837-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-782295199_22 at /127.0.0.1:55244 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=504 (was 338) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7610 (was 8023) 2024-11-13T05:40:16,740 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=504, ProcessCount=11, AvailableMemoryMB=7603 2024-11-13T05:40:16,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:40:16,742 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:40:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:16,744 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:40:16,745 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:16,745 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 24 2024-11-13T05:40:16,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T05:40:16,746 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:40:16,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741951_1127 (size=959) 2024-11-13T05:40:16,764 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745 2024-11-13T05:40:16,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741952_1128 (size=53) 2024-11-13T05:40:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T05:40:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T05:40:17,178 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:40:17,179 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2aab469b292723a934ce524f70b46f92, disabling compactions & flushes 2024-11-13T05:40:17,179 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:17,179 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:17,179 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. after waiting 0 ms 2024-11-13T05:40:17,179 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:17,179 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:17,179 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2aab469b292723a934ce524f70b46f92: Waiting for close lock at 1731476417179Disabling compacts and flushes for region at 1731476417179Disabling writes for close at 1731476417179Writing region close event to WAL at 1731476417179Closed at 1731476417179 2024-11-13T05:40:17,181 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:40:17,182 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731476417181"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731476417181"}]},"ts":"1731476417181"} 2024-11-13T05:40:17,184 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:40:17,186 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:40:17,186 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476417186"}]},"ts":"1731476417186"} 2024-11-13T05:40:17,188 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T05:40:17,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, ASSIGN}] 2024-11-13T05:40:17,190 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, ASSIGN 2024-11-13T05:40:17,191 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45837,1731476380128; forceNewPlan=false, retain=false 2024-11-13T05:40:17,343 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=2aab469b292723a934ce524f70b46f92, regionState=OPENING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:17,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, ASSIGN because future has completed 2024-11-13T05:40:17,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:40:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T05:40:17,512 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:17,512 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7752): Opening region: {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:40:17,513 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:17,513 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:40:17,513 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7794): checking encryption for 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:17,513 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7797): checking classloading for 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:17,517 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:17,520 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2aab469b292723a934ce524f70b46f92 columnFamilyName A 2024-11-13T05:40:17,520 DEBUG [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:17,521 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] regionserver.HStore(327): Store=2aab469b292723a934ce524f70b46f92/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:40:17,521 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:17,522 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2aab469b292723a934ce524f70b46f92 columnFamilyName B 2024-11-13T05:40:17,523 DEBUG [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:17,523 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] regionserver.HStore(327): Store=2aab469b292723a934ce524f70b46f92/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:40:17,523 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:17,525 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2aab469b292723a934ce524f70b46f92 columnFamilyName C 2024-11-13T05:40:17,525 DEBUG [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:17,526 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] regionserver.HStore(327): Store=2aab469b292723a934ce524f70b46f92/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:40:17,526 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1038): replaying wal for 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:17,527 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:17,528 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:17,529 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1048): stopping wal replay for 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:17,529 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1060): Cleaning up temporary data for 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:17,530 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:40:17,532 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): writing seq id for 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:17,535 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:40:17,536 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1114): Opened 2aab469b292723a934ce524f70b46f92; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75240072, jitterRate=0.12116444110870361}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:40:17,536 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:17,538 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1006): Region open journal for 2aab469b292723a934ce524f70b46f92: Running coprocessor pre-open hook at 1731476417514Writing region info on filesystem at 1731476417514Initializing all the Stores at 1731476417515 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476417516 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476417516Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476417516Cleaning up temporary data from old regions at 1731476417529 (+13 ms)Running coprocessor post-open hooks at 1731476417536 (+7 ms)Region opened successfully at 1731476417537 (+1 ms) 2024-11-13T05:40:17,539 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., pid=26, masterSystemTime=1731476417504 2024-11-13T05:40:17,542 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:17,542 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:17,542 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=2aab469b292723a934ce524f70b46f92, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:17,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:40:17,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-13T05:40:17,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 in 196 msec 2024-11-13T05:40:17,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-13T05:40:17,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, ASSIGN in 359 msec 2024-11-13T05:40:17,552 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:40:17,552 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476417552"}]},"ts":"1731476417552"} 2024-11-13T05:40:17,554 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T05:40:17,556 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:40:17,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 814 msec 2024-11-13T05:40:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-13T05:40:17,887 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:17,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@244f1d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:17,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:17,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:17,891 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac03e2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:17,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:17,893 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a9145e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:17,895 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:17,898 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:17,900 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:40:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,37913,1731476379290 2024-11-13T05:40:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33e75818 2024-11-13T05:40:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:40:17,902 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:40:17,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:40:17,908 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:40:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:17,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741953_1129 (size=995) 2024-11-13T05:40:18,329 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-13T05:40:18,329 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-13T05:40:18,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:40:18,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, REOPEN/MOVE}] 2024-11-13T05:40:18,341 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, REOPEN/MOVE 2024-11-13T05:40:18,341 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=2aab469b292723a934ce524f70b46f92, regionState=CLOSING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:18,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, REOPEN/MOVE because future has completed 2024-11-13T05:40:18,343 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:40:18,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:40:18,496 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,496 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:40:18,496 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 2aab469b292723a934ce524f70b46f92, disabling compactions & flushes 2024-11-13T05:40:18,496 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:18,496 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:18,496 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. after waiting 0 ms 2024-11-13T05:40:18,496 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:18,501 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:40:18,502 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:18,502 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 2aab469b292723a934ce524f70b46f92: Waiting for close lock at 1731476418496Running coprocessor pre-close hooks at 1731476418496Disabling compacts and flushes for region at 1731476418496Disabling writes for close at 1731476418496Writing region close event to WAL at 1731476418497 (+1 ms)Running coprocessor post-close hooks at 1731476418502 (+5 ms)Closed at 1731476418502 2024-11-13T05:40:18,502 WARN [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionServer(3299): Not adding moved region record: 2aab469b292723a934ce524f70b46f92 to self. 2024-11-13T05:40:18,505 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,505 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=2aab469b292723a934ce524f70b46f92, regionState=CLOSED 2024-11-13T05:40:18,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:40:18,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-13T05:40:18,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 in 165 msec 2024-11-13T05:40:18,511 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, REOPEN/MOVE; state=CLOSED, location=bc386b7bf2fe,45837,1731476380128; forceNewPlan=false, retain=true 2024-11-13T05:40:18,662 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=2aab469b292723a934ce524f70b46f92, regionState=OPENING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:18,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, REOPEN/MOVE because future has completed 2024-11-13T05:40:18,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:40:18,823 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:18,823 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:40:18,824 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,824 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:40:18,824 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,825 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,831 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,838 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2aab469b292723a934ce524f70b46f92 columnFamilyName A 2024-11-13T05:40:18,840 DEBUG [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:18,840 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] regionserver.HStore(327): Store=2aab469b292723a934ce524f70b46f92/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:40:18,841 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,842 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2aab469b292723a934ce524f70b46f92 columnFamilyName B 2024-11-13T05:40:18,842 DEBUG [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:18,842 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] regionserver.HStore(327): Store=2aab469b292723a934ce524f70b46f92/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:40:18,842 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,843 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2aab469b292723a934ce524f70b46f92 columnFamilyName C 2024-11-13T05:40:18,843 DEBUG [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:18,843 INFO [StoreOpener-2aab469b292723a934ce524f70b46f92-1 {}] regionserver.HStore(327): Store=2aab469b292723a934ce524f70b46f92/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:40:18,844 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,844 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,845 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,846 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,846 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,847 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:40:18,848 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,849 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 2aab469b292723a934ce524f70b46f92; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75180301, jitterRate=0.12027378380298615}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:40:18,849 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,852 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 2aab469b292723a934ce524f70b46f92: Running coprocessor pre-open hook at 1731476418825Writing region info on filesystem at 1731476418825Initializing all the Stores at 1731476418827 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476418827Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476418828 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476418828Cleaning up temporary data from old regions at 1731476418846 (+18 ms)Running coprocessor post-open hooks at 1731476418849 (+3 ms)Region opened successfully at 1731476418852 (+3 ms) 2024-11-13T05:40:18,853 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., pid=31, masterSystemTime=1731476418816 2024-11-13T05:40:18,855 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:18,855 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:18,855 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=2aab469b292723a934ce524f70b46f92, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:18,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:40:18,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-13T05:40:18,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 in 194 msec 2024-11-13T05:40:18,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-13T05:40:18,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, REOPEN/MOVE in 519 msec 2024-11-13T05:40:18,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-13T05:40:18,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 530 msec 2024-11-13T05:40:18,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 954 msec 2024-11-13T05:40:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T05:40:18,870 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:18,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b4ffc72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,871 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:18,871 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:18,872 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc156b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:18,874 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:18,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c27ba6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:18,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:18,877 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6490be12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:18,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:18,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:18,879 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74911b16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:18,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:18,881 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@515d24c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:18,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:18,882 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cff06d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208b436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:18,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:18,884 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f408ff6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:18,886 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59cbafb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:18,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:18,887 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205dbceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:18,889 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b2e109, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,890 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:18,890 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:18,891 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7801a810, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:18,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:18,892 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503216be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:18,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:18,894 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:18,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:18,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:18,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5523a5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:18,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:18,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:18,896 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:18,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b80d230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:18,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:18,898 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195fb47c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:18,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:18,899 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3626668d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:18,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:18,901 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf982c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:18,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:18,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:18,902 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e4f9348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:18,904 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:18,904 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:18,904 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:18,904 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:18,905 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:18,905 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees 2024-11-13T05:40:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:18,905 DEBUG [Thread-655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:18,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:18,906 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T05:40:18,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:18,907 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:18,908 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:18,908 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:18,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:18,909 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:18,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:18,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:18,911 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:18,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:18,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:40:18,911 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:40:18,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:40:18,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:18,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:18,913 DEBUG [Thread-664 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:18,914 DEBUG [Thread-664 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:40:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:40:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:40:18,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:40:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:40:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:40:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:18,928 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:18,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:40:18,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476478975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:18,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476478975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:18,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476478977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:18,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476478977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:18,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476478974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:18,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:18,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:18,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:18,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:18,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:18,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111310df4568ec7a4539af0c9be6d604dafa_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476418928/Put/seqid=0 2024-11-13T05:40:19,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741954_1130 (size=14594) 2024-11-13T05:40:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T05:40:19,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:19,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:19,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:19,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:19,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476479100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476479100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476479101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476479101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476479101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:19,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T05:40:19,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476479316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476479316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476479317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476479317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476479317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:19,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:19,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:19,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:19,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,407 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:19,413 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111310df4568ec7a4539af0c9be6d604dafa_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111310df4568ec7a4539af0c9be6d604dafa_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:19,414 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/b12486a1f002458f9eeb758aa7cab94e, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:19,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/b12486a1f002458f9eeb758aa7cab94e is 175, key is test_row_0/A:col10/1731476418928/Put/seqid=0 2024-11-13T05:40:19,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741955_1131 (size=39549) 2024-11-13T05:40:19,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:19,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:19,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:19,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:19,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T05:40:19,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476479627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476479627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476479627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476479637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476479637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:19,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:19,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:19,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:19,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:19,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:19,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,879 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/b12486a1f002458f9eeb758aa7cab94e 2024-11-13T05:40:19,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/74855d58134549389a45ac046294f2f6 is 50, key is test_row_0/B:col10/1731476418928/Put/seqid=0 2024-11-13T05:40:19,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741956_1132 (size=12001) 2024-11-13T05:40:19,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/74855d58134549389a45ac046294f2f6 2024-11-13T05:40:19,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:19,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:19,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:19,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:19,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/2843ad337531427ca8b6910fcf18dc9f is 50, key is test_row_0/C:col10/1731476418928/Put/seqid=0 2024-11-13T05:40:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T05:40:20,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741957_1133 (size=12001) 2024-11-13T05:40:20,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/2843ad337531427ca8b6910fcf18dc9f 2024-11-13T05:40:20,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/b12486a1f002458f9eeb758aa7cab94e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b12486a1f002458f9eeb758aa7cab94e 2024-11-13T05:40:20,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b12486a1f002458f9eeb758aa7cab94e, entries=200, sequenceid=16, filesize=38.6 K 2024-11-13T05:40:20,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/74855d58134549389a45ac046294f2f6 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/74855d58134549389a45ac046294f2f6 2024-11-13T05:40:20,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/74855d58134549389a45ac046294f2f6, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T05:40:20,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/2843ad337531427ca8b6910fcf18dc9f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/2843ad337531427ca8b6910fcf18dc9f 2024-11-13T05:40:20,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/2843ad337531427ca8b6910fcf18dc9f, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T05:40:20,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2aab469b292723a934ce524f70b46f92 in 1162ms, sequenceid=16, compaction requested=false 2024-11-13T05:40:20,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:20,142 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:20,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T05:40:20,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:20,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:20,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:20,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:20,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a3a50d37872548e183ec6e7a7ae2f935_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476420142/Put/seqid=0 2024-11-13T05:40:20,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476480154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476480156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476480158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476480160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476480162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741958_1134 (size=17034) 2024-11-13T05:40:20,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476480267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476480267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476480267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476480277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476480279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:20,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:20,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:20,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:20,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:20,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:20,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:20,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:20,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476480477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476480486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476480486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476480486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476480487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,592 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:20,602 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a3a50d37872548e183ec6e7a7ae2f935_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a3a50d37872548e183ec6e7a7ae2f935_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:20,605 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/9939e5ea1be84943a76a88d5547cd487, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:20,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/9939e5ea1be84943a76a88d5547cd487 is 175, key is test_row_0/A:col10/1731476420142/Put/seqid=0 2024-11-13T05:40:20,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:20,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:20,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:20,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:20,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741959_1135 (size=48139) 2024-11-13T05:40:20,661 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/9939e5ea1be84943a76a88d5547cd487 2024-11-13T05:40:20,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/3e723dd5e41f42f3be5c0ffcd39fe5ea is 50, key is test_row_0/B:col10/1731476420142/Put/seqid=0 2024-11-13T05:40:20,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741960_1136 (size=12001) 2024-11-13T05:40:20,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/3e723dd5e41f42f3be5c0ffcd39fe5ea 2024-11-13T05:40:20,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/84508c151e8346aba55e13349ee14e57 is 50, key is test_row_0/C:col10/1731476420142/Put/seqid=0 2024-11-13T05:40:20,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:20,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:20,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:20,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:20,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476480786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476480797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476480797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741961_1137 (size=12001) 2024-11-13T05:40:20,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476480797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:20,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476480806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:20,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:20,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T05:40:21,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:21,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:21,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:21,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:21,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:21,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:21,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/84508c151e8346aba55e13349ee14e57 2024-11-13T05:40:21,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/9939e5ea1be84943a76a88d5547cd487 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/9939e5ea1be84943a76a88d5547cd487 2024-11-13T05:40:21,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/9939e5ea1be84943a76a88d5547cd487, entries=250, sequenceid=42, filesize=47.0 K 2024-11-13T05:40:21,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/3e723dd5e41f42f3be5c0ffcd39fe5ea as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/3e723dd5e41f42f3be5c0ffcd39fe5ea 2024-11-13T05:40:21,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:21,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:21,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:21,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:21,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:21,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:21,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:21,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/3e723dd5e41f42f3be5c0ffcd39fe5ea, entries=150, sequenceid=42, filesize=11.7 K 2024-11-13T05:40:21,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/84508c151e8346aba55e13349ee14e57 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/84508c151e8346aba55e13349ee14e57 2024-11-13T05:40:21,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/84508c151e8346aba55e13349ee14e57, entries=150, sequenceid=42, filesize=11.7 K 2024-11-13T05:40:21,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 2aab469b292723a934ce524f70b46f92 in 1096ms, sequenceid=42, compaction requested=false 2024-11-13T05:40:21,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:21,311 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:21,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:40:21,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138b926bc8d5e74d5492b94ef6f4a78197_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476420159/Put/seqid=0 2024-11-13T05:40:21,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476481351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476481351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476481352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476481357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476481360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741962_1138 (size=14594) 2024-11-13T05:40:21,374 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:21,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:21,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:21,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:21,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:21,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:21,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:21,382 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138b926bc8d5e74d5492b94ef6f4a78197_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138b926bc8d5e74d5492b94ef6f4a78197_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:21,384 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/61ef8808520b43b1a4b3c98f6cff072d, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:21,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/61ef8808520b43b1a4b3c98f6cff072d is 175, key is test_row_0/A:col10/1731476420159/Put/seqid=0 2024-11-13T05:40:21,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741963_1139 (size=39549) 2024-11-13T05:40:21,440 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/61ef8808520b43b1a4b3c98f6cff072d 2024-11-13T05:40:21,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/2715aa1684664b6b958aa4d021724a67 is 50, key is test_row_0/B:col10/1731476420159/Put/seqid=0 2024-11-13T05:40:21,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476481466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476481466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476481466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476481477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476481478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741964_1140 (size=12001) 2024-11-13T05:40:21,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/2715aa1684664b6b958aa4d021724a67 2024-11-13T05:40:21,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/684c8def7c40490a99a38ef27b7f1504 is 50, key is test_row_0/C:col10/1731476420159/Put/seqid=0 2024-11-13T05:40:21,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:21,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:21,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:21,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:21,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:21,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741965_1141 (size=12001) 2024-11-13T05:40:21,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/684c8def7c40490a99a38ef27b7f1504 2024-11-13T05:40:21,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/61ef8808520b43b1a4b3c98f6cff072d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/61ef8808520b43b1a4b3c98f6cff072d 2024-11-13T05:40:21,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/61ef8808520b43b1a4b3c98f6cff072d, entries=200, sequenceid=53, filesize=38.6 K 2024-11-13T05:40:21,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/2715aa1684664b6b958aa4d021724a67 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/2715aa1684664b6b958aa4d021724a67 2024-11-13T05:40:21,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/2715aa1684664b6b958aa4d021724a67, entries=150, sequenceid=53, filesize=11.7 K 2024-11-13T05:40:21,596 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:40:21,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/684c8def7c40490a99a38ef27b7f1504 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/684c8def7c40490a99a38ef27b7f1504 2024-11-13T05:40:21,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/684c8def7c40490a99a38ef27b7f1504, entries=150, sequenceid=53, filesize=11.7 K 2024-11-13T05:40:21,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 2aab469b292723a934ce524f70b46f92 in 314ms, sequenceid=53, compaction requested=true 2024-11-13T05:40:21,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:21,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:21,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:21,626 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:21,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:21,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:21,626 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:21,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:21,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:21,627 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:21,627 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:21,627 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/A is initiating minor compaction (all files) 2024-11-13T05:40:21,627 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/B is initiating minor compaction (all files) 2024-11-13T05:40:21,628 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/A in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:21,628 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/B in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:21,628 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/74855d58134549389a45ac046294f2f6, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/3e723dd5e41f42f3be5c0ffcd39fe5ea, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/2715aa1684664b6b958aa4d021724a67] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=35.2 K 2024-11-13T05:40:21,628 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b12486a1f002458f9eeb758aa7cab94e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/9939e5ea1be84943a76a88d5547cd487, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/61ef8808520b43b1a4b3c98f6cff072d] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=124.3 K 2024-11-13T05:40:21,628 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:21,628 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b12486a1f002458f9eeb758aa7cab94e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/9939e5ea1be84943a76a88d5547cd487, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/61ef8808520b43b1a4b3c98f6cff072d] 2024-11-13T05:40:21,629 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 74855d58134549389a45ac046294f2f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731476418925 2024-11-13T05:40:21,629 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting b12486a1f002458f9eeb758aa7cab94e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731476418925 2024-11-13T05:40:21,629 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e723dd5e41f42f3be5c0ffcd39fe5ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731476418974 2024-11-13T05:40:21,629 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9939e5ea1be84943a76a88d5547cd487, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731476418970 2024-11-13T05:40:21,630 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61ef8808520b43b1a4b3c98f6cff072d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731476420151 2024-11-13T05:40:21,630 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 2715aa1684664b6b958aa4d021724a67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731476420155 2024-11-13T05:40:21,653 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:21,654 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/8c500af0fe5f4e1db96427f5444a1f91 is 50, key is test_row_0/B:col10/1731476420159/Put/seqid=0 2024-11-13T05:40:21,671 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:21,683 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411136e54f4d4204d4465a2f89524961b198d_2aab469b292723a934ce524f70b46f92 store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:21,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-13T05:40:21,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:21,688 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T05:40:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:21,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:21,696 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411136e54f4d4204d4465a2f89524961b198d_2aab469b292723a934ce524f70b46f92, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:21,697 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136e54f4d4204d4465a2f89524961b198d_2aab469b292723a934ce524f70b46f92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:21,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476481700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476481709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476481711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476481711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476481715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741966_1142 (size=12104) 2024-11-13T05:40:21,745 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/8c500af0fe5f4e1db96427f5444a1f91 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/8c500af0fe5f4e1db96427f5444a1f91 2024-11-13T05:40:21,753 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/B of 2aab469b292723a934ce524f70b46f92 into 8c500af0fe5f4e1db96427f5444a1f91(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:21,753 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:21,753 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/B, priority=13, startTime=1731476421626; duration=0sec 2024-11-13T05:40:21,753 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:21,753 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:B 2024-11-13T05:40:21,753 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e60529325eb044b89dae44f5cfafd99e_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476421355/Put/seqid=0 2024-11-13T05:40:21,756 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:21,757 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/C is initiating minor compaction (all files) 2024-11-13T05:40:21,757 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/C in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:21,757 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/2843ad337531427ca8b6910fcf18dc9f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/84508c151e8346aba55e13349ee14e57, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/684c8def7c40490a99a38ef27b7f1504] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=35.2 K 2024-11-13T05:40:21,758 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 2843ad337531427ca8b6910fcf18dc9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731476418925 2024-11-13T05:40:21,758 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 84508c151e8346aba55e13349ee14e57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731476418974 2024-11-13T05:40:21,759 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 684c8def7c40490a99a38ef27b7f1504, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731476420155 2024-11-13T05:40:21,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741967_1143 (size=4469) 2024-11-13T05:40:21,797 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:21,798 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/86df8e0def9b4a7f96e6fa6df6ecf42d is 50, key is test_row_0/C:col10/1731476420159/Put/seqid=0 2024-11-13T05:40:21,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741968_1144 (size=12154) 2024-11-13T05:40:21,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476481816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476481827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476481826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476481827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476481841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:21,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741969_1145 (size=12104) 2024-11-13T05:40:21,856 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/86df8e0def9b4a7f96e6fa6df6ecf42d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/86df8e0def9b4a7f96e6fa6df6ecf42d 2024-11-13T05:40:21,865 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/C of 2aab469b292723a934ce524f70b46f92 into 86df8e0def9b4a7f96e6fa6df6ecf42d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:21,865 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:21,865 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/C, priority=13, startTime=1731476421626; duration=0sec 2024-11-13T05:40:21,865 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:21,865 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:C 2024-11-13T05:40:22,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476482036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476482046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476482046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476482047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476482047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,186 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#A#compaction#124 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:22,187 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/58c36ea653cc490888b001b3369fc314 is 175, key is test_row_0/A:col10/1731476420159/Put/seqid=0 2024-11-13T05:40:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:22,233 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e60529325eb044b89dae44f5cfafd99e_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e60529325eb044b89dae44f5cfafd99e_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:22,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/afecfa1117d24e3987d24632fa045495, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:22,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/afecfa1117d24e3987d24632fa045495 is 175, key is test_row_0/A:col10/1731476421355/Put/seqid=0 2024-11-13T05:40:22,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741970_1146 (size=31058) 2024-11-13T05:40:22,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741971_1147 (size=30955) 2024-11-13T05:40:22,256 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/afecfa1117d24e3987d24632fa045495 2024-11-13T05:40:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/3f395de68f284337b1db66ac44195a17 is 50, key is test_row_0/B:col10/1731476421355/Put/seqid=0 2024-11-13T05:40:22,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741972_1148 (size=12001) 2024-11-13T05:40:22,353 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/3f395de68f284337b1db66ac44195a17 2024-11-13T05:40:22,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476482358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476482358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476482358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476482358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476482358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/d143e3e520aa4eaa9e22703b030ea99b is 50, key is test_row_0/C:col10/1731476421355/Put/seqid=0 2024-11-13T05:40:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741973_1149 (size=12001) 2024-11-13T05:40:22,433 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/d143e3e520aa4eaa9e22703b030ea99b 2024-11-13T05:40:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/afecfa1117d24e3987d24632fa045495 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/afecfa1117d24e3987d24632fa045495 2024-11-13T05:40:22,449 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/afecfa1117d24e3987d24632fa045495, entries=150, sequenceid=79, filesize=30.2 K 2024-11-13T05:40:22,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/3f395de68f284337b1db66ac44195a17 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/3f395de68f284337b1db66ac44195a17 2024-11-13T05:40:22,461 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/3f395de68f284337b1db66ac44195a17, entries=150, sequenceid=79, filesize=11.7 K 2024-11-13T05:40:22,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/d143e3e520aa4eaa9e22703b030ea99b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d143e3e520aa4eaa9e22703b030ea99b 2024-11-13T05:40:22,472 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d143e3e520aa4eaa9e22703b030ea99b, entries=150, sequenceid=79, filesize=11.7 K 2024-11-13T05:40:22,474 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 2aab469b292723a934ce524f70b46f92 in 785ms, sequenceid=79, compaction requested=false 2024-11-13T05:40:22,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:22,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:22,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-13T05:40:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-13T05:40:22,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-13T05:40:22,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5680 sec 2024-11-13T05:40:22,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees in 3.5750 sec 2024-11-13T05:40:22,647 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/58c36ea653cc490888b001b3369fc314 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/58c36ea653cc490888b001b3369fc314 2024-11-13T05:40:22,656 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/A of 2aab469b292723a934ce524f70b46f92 into 58c36ea653cc490888b001b3369fc314(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-13T05:40:22,656 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:22,656 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/A, priority=13, startTime=1731476421626; duration=1sec 2024-11-13T05:40:22,656 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:22,656 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:A 2024-11-13T05:40:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:22,870 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:22,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T05:40:22,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113295553f501c74e6f9aa59a8ba9132d04_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476421706/Put/seqid=0 2024-11-13T05:40:22,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741974_1150 (size=14594) 2024-11-13T05:40:22,914 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:22,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476482902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476482904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476482904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476482909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476482906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:22,921 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113295553f501c74e6f9aa59a8ba9132d04_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113295553f501c74e6f9aa59a8ba9132d04_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:22,923 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/20e3ec5b8f8848e395659cf43a01e208, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:22,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/20e3ec5b8f8848e395659cf43a01e208 is 175, key is test_row_0/A:col10/1731476421706/Put/seqid=0 2024-11-13T05:40:22,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741975_1151 (size=39549) 2024-11-13T05:40:23,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476483027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476483027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476483027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476483027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476483027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T05:40:23,077 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:23,086 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-13T05:40:23,089 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T05:40:23,090 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:23,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T05:40:23,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T05:40:23,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:23,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:23,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:23,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476483237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476483237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476483237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476483238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476483238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,339 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/20e3ec5b8f8848e395659cf43a01e208 2024-11-13T05:40:23,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/e6dc686fbfb648cea2ea4e7b4e687978 is 50, key is test_row_0/B:col10/1731476421706/Put/seqid=0 2024-11-13T05:40:23,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T05:40:23,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:23,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:23,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:23,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T05:40:23,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741976_1152 (size=12001) 2024-11-13T05:40:23,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T05:40:23,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:23,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:23,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:23,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476483556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476483557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476483558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476483566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476483567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:23,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:23,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T05:40:23,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T05:40:23,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:23,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:23,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:23,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/e6dc686fbfb648cea2ea4e7b4e687978 2024-11-13T05:40:23,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/de69703fd4194f139009af8f39fefb31 is 50, key is test_row_0/C:col10/1731476421706/Put/seqid=0 2024-11-13T05:40:23,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741977_1153 (size=12001) 2024-11-13T05:40:23,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/de69703fd4194f139009af8f39fefb31 2024-11-13T05:40:23,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/20e3ec5b8f8848e395659cf43a01e208 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/20e3ec5b8f8848e395659cf43a01e208 2024-11-13T05:40:23,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/20e3ec5b8f8848e395659cf43a01e208, entries=200, sequenceid=96, filesize=38.6 K 2024-11-13T05:40:23,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/e6dc686fbfb648cea2ea4e7b4e687978 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/e6dc686fbfb648cea2ea4e7b4e687978 2024-11-13T05:40:23,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T05:40:23,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:23,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:23,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:23,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:23,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/e6dc686fbfb648cea2ea4e7b4e687978, entries=150, sequenceid=96, filesize=11.7 K 2024-11-13T05:40:23,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/de69703fd4194f139009af8f39fefb31 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/de69703fd4194f139009af8f39fefb31 2024-11-13T05:40:23,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/de69703fd4194f139009af8f39fefb31, entries=150, sequenceid=96, filesize=11.7 K 2024-11-13T05:40:23,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 2aab469b292723a934ce524f70b46f92 in 1030ms, sequenceid=96, compaction requested=true 2024-11-13T05:40:23,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:23,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:23,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:23,900 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:23,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:23,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:23,900 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:23,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:23,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:23,901 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:23,901 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/B is initiating minor compaction (all files) 2024-11-13T05:40:23,902 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:23,902 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/A is initiating minor compaction (all files) 2024-11-13T05:40:23,902 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/A in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:23,902 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/58c36ea653cc490888b001b3369fc314, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/afecfa1117d24e3987d24632fa045495, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/20e3ec5b8f8848e395659cf43a01e208] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=99.2 K 2024-11-13T05:40:23,902 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:23,902 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/58c36ea653cc490888b001b3369fc314, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/afecfa1117d24e3987d24632fa045495, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/20e3ec5b8f8848e395659cf43a01e208] 2024-11-13T05:40:23,903 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/B in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:23,904 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/8c500af0fe5f4e1db96427f5444a1f91, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/3f395de68f284337b1db66ac44195a17, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/e6dc686fbfb648cea2ea4e7b4e687978] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=35.3 K 2024-11-13T05:40:23,904 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58c36ea653cc490888b001b3369fc314, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731476420155 2024-11-13T05:40:23,904 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c500af0fe5f4e1db96427f5444a1f91, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731476420155 2024-11-13T05:40:23,904 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting afecfa1117d24e3987d24632fa045495, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731476421346 2024-11-13T05:40:23,905 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f395de68f284337b1db66ac44195a17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731476421346 2024-11-13T05:40:23,905 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20e3ec5b8f8848e395659cf43a01e208, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731476421706 2024-11-13T05:40:23,905 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting e6dc686fbfb648cea2ea4e7b4e687978, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731476421706 2024-11-13T05:40:23,924 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:23,926 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#B#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:23,926 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/e186c0bfa011498c9824e15f9bb0a1cc is 50, key is test_row_0/B:col10/1731476421706/Put/seqid=0 2024-11-13T05:40:23,928 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113553c0947c707491f9110b77c7cac3df1_2aab469b292723a934ce524f70b46f92 store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:23,932 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113553c0947c707491f9110b77c7cac3df1_2aab469b292723a934ce524f70b46f92, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:23,933 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113553c0947c707491f9110b77c7cac3df1_2aab469b292723a934ce524f70b46f92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:23,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741978_1154 (size=12207) 2024-11-13T05:40:23,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741979_1155 (size=4469) 2024-11-13T05:40:24,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-13T05:40:24,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:24,034 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T05:40:24,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c00f5a8347e844638997b13e0e9766ff_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476422902/Put/seqid=0 2024-11-13T05:40:24,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741980_1156 (size=12154) 2024-11-13T05:40:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:24,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:24,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476484087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:24,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476484087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:24,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:24,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476484087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:24,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:24,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476484088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:24,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:24,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476484091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:24,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476484196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476484196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476484196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T05:40:24,344 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#A#compaction#133 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:24,345 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/b454374a60ec41eca51970911a6a5ff0 is 175, key is test_row_0/A:col10/1731476421706/Put/seqid=0 2024-11-13T05:40:24,349 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/e186c0bfa011498c9824e15f9bb0a1cc as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/e186c0bfa011498c9824e15f9bb0a1cc 2024-11-13T05:40:24,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741981_1157 (size=31161) 2024-11-13T05:40:24,356 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/B of 2aab469b292723a934ce524f70b46f92 into e186c0bfa011498c9824e15f9bb0a1cc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:24,356 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:24,356 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/B, priority=13, startTime=1731476423900; duration=0sec 2024-11-13T05:40:24,356 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:24,357 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:B 2024-11-13T05:40:24,357 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:24,359 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:24,359 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/C is initiating minor compaction (all files) 2024-11-13T05:40:24,359 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/C in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:24,359 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/86df8e0def9b4a7f96e6fa6df6ecf42d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d143e3e520aa4eaa9e22703b030ea99b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/de69703fd4194f139009af8f39fefb31] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=35.3 K 2024-11-13T05:40:24,359 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 86df8e0def9b4a7f96e6fa6df6ecf42d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731476420155 2024-11-13T05:40:24,360 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting d143e3e520aa4eaa9e22703b030ea99b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731476421346 2024-11-13T05:40:24,360 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting de69703fd4194f139009af8f39fefb31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731476421706 2024-11-13T05:40:24,366 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#C#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:24,367 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/d8df68e7d22f4188a3e2d84d52fa47f9 is 50, key is test_row_0/C:col10/1731476421706/Put/seqid=0 2024-11-13T05:40:24,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741982_1158 (size=12207) 2024-11-13T05:40:24,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476484406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:24,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:24,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476484407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:24,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476484407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:24,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:24,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:24,461 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c00f5a8347e844638997b13e0e9766ff_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c00f5a8347e844638997b13e0e9766ff_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:24,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/6812399ba5494fb7a2c6ff0ad8386656, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:24,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/6812399ba5494fb7a2c6ff0ad8386656 is 175, key is test_row_0/A:col10/1731476422902/Put/seqid=0 2024-11-13T05:40:24,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741983_1159 (size=30955) 2024-11-13T05:40:24,472 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/6812399ba5494fb7a2c6ff0ad8386656 2024-11-13T05:40:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/0df6b336cf9d49aeac68a8028620f12c is 50, key is test_row_0/B:col10/1731476422902/Put/seqid=0 2024-11-13T05:40:24,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741984_1160 (size=12001) 2024-11-13T05:40:24,508 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/0df6b336cf9d49aeac68a8028620f12c 2024-11-13T05:40:24,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/4c356dede88f4550957df0285712bbbb is 50, key is test_row_0/C:col10/1731476422902/Put/seqid=0 2024-11-13T05:40:24,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741985_1161 (size=12001) 2024-11-13T05:40:24,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476484717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:24,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476484717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:24,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:24,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476484726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:24,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:24,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:24,758 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/b454374a60ec41eca51970911a6a5ff0 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b454374a60ec41eca51970911a6a5ff0 2024-11-13T05:40:24,763 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/A of 2aab469b292723a934ce524f70b46f92 into b454374a60ec41eca51970911a6a5ff0(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:24,763 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:24,763 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/A, priority=13, startTime=1731476423900; duration=0sec 2024-11-13T05:40:24,763 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:24,763 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:A 2024-11-13T05:40:24,777 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/d8df68e7d22f4188a3e2d84d52fa47f9 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d8df68e7d22f4188a3e2d84d52fa47f9 2024-11-13T05:40:24,785 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/C of 2aab469b292723a934ce524f70b46f92 into d8df68e7d22f4188a3e2d84d52fa47f9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:24,785 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:24,785 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/C, priority=13, startTime=1731476423900; duration=0sec 2024-11-13T05:40:24,785 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:24,785 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:C 2024-11-13T05:40:24,930 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/4c356dede88f4550957df0285712bbbb 2024-11-13T05:40:24,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/6812399ba5494fb7a2c6ff0ad8386656 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/6812399ba5494fb7a2c6ff0ad8386656 2024-11-13T05:40:24,939 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/6812399ba5494fb7a2c6ff0ad8386656, entries=150, sequenceid=118, filesize=30.2 K 2024-11-13T05:40:24,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/0df6b336cf9d49aeac68a8028620f12c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/0df6b336cf9d49aeac68a8028620f12c 2024-11-13T05:40:24,949 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/0df6b336cf9d49aeac68a8028620f12c, entries=150, sequenceid=118, filesize=11.7 K 2024-11-13T05:40:24,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/4c356dede88f4550957df0285712bbbb as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/4c356dede88f4550957df0285712bbbb 2024-11-13T05:40:24,953 INFO [master/bc386b7bf2fe:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T05:40:24,953 INFO [master/bc386b7bf2fe:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T05:40:24,959 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/4c356dede88f4550957df0285712bbbb, entries=150, sequenceid=118, filesize=11.7 K 2024-11-13T05:40:24,959 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 2aab469b292723a934ce524f70b46f92 in 926ms, sequenceid=118, compaction requested=false 2024-11-13T05:40:24,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:24,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:24,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-13T05:40:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-13T05:40:24,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-13T05:40:24,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8720 sec 2024-11-13T05:40:24,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 1.8780 sec 2024-11-13T05:40:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:25,098 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:25,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-13T05:40:25,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c60c163e37b84f5ea2f31dc2da6635cc_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476425097/Put/seqid=0 2024-11-13T05:40:25,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741986_1162 (size=14744) 2024-11-13T05:40:25,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476485152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:25,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:25,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476485153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:25,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:25,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476485227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:25,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476485227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T05:40:25,236 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:25,237 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-13T05:40:25,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T05:40:25,239 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:25,239 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:25,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:25,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476485246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:25,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476485256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:25,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476485267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:25,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T05:40:25,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T05:40:25,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:25,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:25,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:25,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:25,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:25,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476485467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:25,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476485476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:25,512 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:25,517 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c60c163e37b84f5ea2f31dc2da6635cc_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c60c163e37b84f5ea2f31dc2da6635cc_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:25,518 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/acd1743d3ee7499d983e8ea6709f04d7, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:25,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/acd1743d3ee7499d983e8ea6709f04d7 is 175, key is test_row_0/A:col10/1731476425097/Put/seqid=0 2024-11-13T05:40:25,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741987_1163 (size=39699) 2024-11-13T05:40:25,538 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/acd1743d3ee7499d983e8ea6709f04d7 2024-11-13T05:40:25,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T05:40:25,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:25,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:25,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:25,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:25,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:25,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/be35989a9d02441c8f7c04d7c1d9d9dc is 50, key is test_row_0/B:col10/1731476425097/Put/seqid=0 2024-11-13T05:40:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T05:40:25,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741988_1164 (size=12101) 2024-11-13T05:40:25,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T05:40:25,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:25,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:25,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:25,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:25,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:25,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476485776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:25,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476485797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:25,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:25,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:25,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T05:40:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:25,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T05:40:25,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/be35989a9d02441c8f7c04d7c1d9d9dc 2024-11-13T05:40:25,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/92dd2875e8b74fcf9c3ba4df1f704f6b is 50, key is test_row_0/C:col10/1731476425097/Put/seqid=0 2024-11-13T05:40:25,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741989_1165 (size=12101) 2024-11-13T05:40:26,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T05:40:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:26,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:26,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T05:40:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:26,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:26,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476486246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:26,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476486246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:26,250 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_0' in 2aab469b292723a934ce524f70b46f92 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:26,250 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in 2aab469b292723a934ce524f70b46f92 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:26,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476486266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:26,270 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_0' in 2aab469b292723a934ce524f70b46f92 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:26,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476486296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:26,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:26,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:26,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476486306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:26,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T05:40:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:26,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T05:40:26,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/92dd2875e8b74fcf9c3ba4df1f704f6b 2024-11-13T05:40:26,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/acd1743d3ee7499d983e8ea6709f04d7 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/acd1743d3ee7499d983e8ea6709f04d7 2024-11-13T05:40:26,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/acd1743d3ee7499d983e8ea6709f04d7, entries=200, sequenceid=136, filesize=38.8 K 2024-11-13T05:40:26,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/be35989a9d02441c8f7c04d7c1d9d9dc as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/be35989a9d02441c8f7c04d7c1d9d9dc 2024-11-13T05:40:26,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/be35989a9d02441c8f7c04d7c1d9d9dc, entries=150, sequenceid=136, filesize=11.8 K 2024-11-13T05:40:26,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/92dd2875e8b74fcf9c3ba4df1f704f6b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/92dd2875e8b74fcf9c3ba4df1f704f6b 2024-11-13T05:40:26,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/92dd2875e8b74fcf9c3ba4df1f704f6b, entries=150, sequenceid=136, filesize=11.8 K 2024-11-13T05:40:26,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 2aab469b292723a934ce524f70b46f92 in 1325ms, sequenceid=136, compaction requested=true 2024-11-13T05:40:26,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:26,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:26,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:26,424 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:26,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:26,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:26,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:26,424 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:26,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:26,425 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:26,425 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:26,425 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/B is initiating minor compaction (all files) 2024-11-13T05:40:26,425 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/A is initiating minor compaction (all files) 2024-11-13T05:40:26,425 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/A in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:26,425 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/B in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:26,425 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b454374a60ec41eca51970911a6a5ff0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/6812399ba5494fb7a2c6ff0ad8386656, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/acd1743d3ee7499d983e8ea6709f04d7] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=99.4 K 2024-11-13T05:40:26,425 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/e186c0bfa011498c9824e15f9bb0a1cc, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/0df6b336cf9d49aeac68a8028620f12c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/be35989a9d02441c8f7c04d7c1d9d9dc] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=35.5 K 2024-11-13T05:40:26,425 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:26,425 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b454374a60ec41eca51970911a6a5ff0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/6812399ba5494fb7a2c6ff0ad8386656, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/acd1743d3ee7499d983e8ea6709f04d7] 2024-11-13T05:40:26,426 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting e186c0bfa011498c9824e15f9bb0a1cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731476421706 2024-11-13T05:40:26,426 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting b454374a60ec41eca51970911a6a5ff0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731476421706 2024-11-13T05:40:26,426 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 0df6b336cf9d49aeac68a8028620f12c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731476422902 2024-11-13T05:40:26,426 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6812399ba5494fb7a2c6ff0ad8386656, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731476422902 2024-11-13T05:40:26,426 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting be35989a9d02441c8f7c04d7c1d9d9dc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731476424078 2024-11-13T05:40:26,427 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting acd1743d3ee7499d983e8ea6709f04d7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731476424078 2024-11-13T05:40:26,435 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:26,436 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:26,436 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/6f96749bdd3c47bdafdd36fe2148b43c is 50, key is test_row_0/B:col10/1731476425097/Put/seqid=0 2024-11-13T05:40:26,437 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113624448279f36473b97dea35742d16942_2aab469b292723a934ce524f70b46f92 store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:26,440 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113624448279f36473b97dea35742d16942_2aab469b292723a934ce524f70b46f92, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:26,440 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113624448279f36473b97dea35742d16942_2aab469b292723a934ce524f70b46f92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:26,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741990_1166 (size=12409) 2024-11-13T05:40:26,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741991_1167 (size=4469) 2024-11-13T05:40:26,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T05:40:26,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:26,464 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T05:40:26,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138a4396f2fdaf457cb27e2eba3e0f6f1a_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476425143/Put/seqid=0 2024-11-13T05:40:26,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741992_1168 (size=12304) 2024-11-13T05:40:26,852 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#A#compaction#142 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:26,853 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/d8d8e390314a47d8b3a5003e561ce355 is 175, key is test_row_0/A:col10/1731476425097/Put/seqid=0 2024-11-13T05:40:26,856 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/6f96749bdd3c47bdafdd36fe2148b43c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/6f96749bdd3c47bdafdd36fe2148b43c 2024-11-13T05:40:26,866 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/B of 2aab469b292723a934ce524f70b46f92 into 6f96749bdd3c47bdafdd36fe2148b43c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:26,866 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:26,866 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/B, priority=13, startTime=1731476426424; duration=0sec 2024-11-13T05:40:26,866 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:26,866 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:B 2024-11-13T05:40:26,867 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:26,870 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:26,870 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/C is initiating minor compaction (all files) 2024-11-13T05:40:26,870 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/C in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:26,870 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d8df68e7d22f4188a3e2d84d52fa47f9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/4c356dede88f4550957df0285712bbbb, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/92dd2875e8b74fcf9c3ba4df1f704f6b] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=35.5 K 2024-11-13T05:40:26,871 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting d8df68e7d22f4188a3e2d84d52fa47f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731476421706 2024-11-13T05:40:26,871 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c356dede88f4550957df0285712bbbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731476422902 2024-11-13T05:40:26,872 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 92dd2875e8b74fcf9c3ba4df1f704f6b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731476424078 2024-11-13T05:40:26,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741993_1169 (size=31363) 2024-11-13T05:40:26,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:26,885 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138a4396f2fdaf457cb27e2eba3e0f6f1a_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138a4396f2fdaf457cb27e2eba3e0f6f1a_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:26,893 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#C#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:26,893 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/d9a0d366460047b4bf4124127ce77167 is 50, key is test_row_0/C:col10/1731476425097/Put/seqid=0 2024-11-13T05:40:26,896 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/d8d8e390314a47d8b3a5003e561ce355 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/d8d8e390314a47d8b3a5003e561ce355 2024-11-13T05:40:26,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/5eb40525177d405babf65bae69e59028, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:26,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/5eb40525177d405babf65bae69e59028 is 175, key is test_row_0/A:col10/1731476425143/Put/seqid=0 2024-11-13T05:40:26,904 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/A of 2aab469b292723a934ce524f70b46f92 into d8d8e390314a47d8b3a5003e561ce355(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:26,904 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:26,904 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/A, priority=13, startTime=1731476426424; duration=0sec 2024-11-13T05:40:26,904 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:26,904 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:A 2024-11-13T05:40:26,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741994_1170 (size=12409) 2024-11-13T05:40:26,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741995_1171 (size=31105) 2024-11-13T05:40:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:27,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:27,323 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/5eb40525177d405babf65bae69e59028 2024-11-13T05:40:27,326 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/d9a0d366460047b4bf4124127ce77167 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d9a0d366460047b4bf4124127ce77167 2024-11-13T05:40:27,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/c1d624e35cf64d2aad4d7f29e6408fad is 50, key is test_row_0/B:col10/1731476425143/Put/seqid=0 2024-11-13T05:40:27,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741996_1172 (size=12151) 2024-11-13T05:40:27,364 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/C of 2aab469b292723a934ce524f70b46f92 into d9a0d366460047b4bf4124127ce77167(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:27,364 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:27,364 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/C, priority=13, startTime=1731476426424; duration=0sec 2024-11-13T05:40:27,364 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:27,364 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:C 2024-11-13T05:40:27,367 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/c1d624e35cf64d2aad4d7f29e6408fad 2024-11-13T05:40:27,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476487374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:27,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:27,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/91d2894a093c4c8090a15d528b568758 is 50, key is test_row_0/C:col10/1731476425143/Put/seqid=0 2024-11-13T05:40:27,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476487375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:27,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741997_1173 (size=12151) 2024-11-13T05:40:27,386 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/91d2894a093c4c8090a15d528b568758 2024-11-13T05:40:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T05:40:27,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/5eb40525177d405babf65bae69e59028 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/5eb40525177d405babf65bae69e59028 2024-11-13T05:40:27,400 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/5eb40525177d405babf65bae69e59028, entries=150, sequenceid=157, filesize=30.4 K 2024-11-13T05:40:27,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/c1d624e35cf64d2aad4d7f29e6408fad as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/c1d624e35cf64d2aad4d7f29e6408fad 2024-11-13T05:40:27,410 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/c1d624e35cf64d2aad4d7f29e6408fad, entries=150, sequenceid=157, filesize=11.9 K 2024-11-13T05:40:27,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/91d2894a093c4c8090a15d528b568758 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/91d2894a093c4c8090a15d528b568758 2024-11-13T05:40:27,419 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/91d2894a093c4c8090a15d528b568758, entries=150, sequenceid=157, filesize=11.9 K 2024-11-13T05:40:27,420 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 2aab469b292723a934ce524f70b46f92 in 956ms, sequenceid=157, compaction requested=false 2024-11-13T05:40:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-13T05:40:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-13T05:40:27,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-13T05:40:27,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1820 sec 2024-11-13T05:40:27,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.1900 sec 2024-11-13T05:40:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:27,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T05:40:27,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d83c2e43cdd1403d9f062a49f576d7db_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476427488/Put/seqid=0 2024-11-13T05:40:27,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741998_1174 (size=17284) 2024-11-13T05:40:27,571 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:27,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476487570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:27,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476487574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:27,578 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d83c2e43cdd1403d9f062a49f576d7db_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d83c2e43cdd1403d9f062a49f576d7db_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:27,580 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/a862f04c3bfa4ec6b16639b52f721c74, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:27,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/a862f04c3bfa4ec6b16639b52f721c74 is 175, key is test_row_0/A:col10/1731476427488/Put/seqid=0 2024-11-13T05:40:27,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741999_1175 (size=48389) 2024-11-13T05:40:27,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476487688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:27,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:27,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476487688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:27,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476487896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:27,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476487896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:27,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:27,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:27,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:28,041 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/a862f04c3bfa4ec6b16639b52f721c74 2024-11-13T05:40:28,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/bf06c24f7b704e0d8374b58186c581e8 is 50, key is test_row_0/B:col10/1731476427488/Put/seqid=0 2024-11-13T05:40:28,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742000_1176 (size=12151) 2024-11-13T05:40:28,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476488206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:28,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476488206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:28,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476488267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:28,273 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in 2aab469b292723a934ce524f70b46f92 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:28,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476488276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:28,279 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_0' in 2aab469b292723a934ce524f70b46f92 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:28,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476488286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:28,294 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_0' in 2aab469b292723a934ce524f70b46f92 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:28,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:28,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/bf06c24f7b704e0d8374b58186c581e8 2024-11-13T05:40:28,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/408dcc5bb46b4075981dfc1797b35079 is 50, key is test_row_0/C:col10/1731476427488/Put/seqid=0 2024-11-13T05:40:28,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742001_1177 (size=12151) 2024-11-13T05:40:28,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/408dcc5bb46b4075981dfc1797b35079 2024-11-13T05:40:28,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/a862f04c3bfa4ec6b16639b52f721c74 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/a862f04c3bfa4ec6b16639b52f721c74 2024-11-13T05:40:28,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/a862f04c3bfa4ec6b16639b52f721c74, entries=250, sequenceid=177, filesize=47.3 K 2024-11-13T05:40:28,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/bf06c24f7b704e0d8374b58186c581e8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/bf06c24f7b704e0d8374b58186c581e8 2024-11-13T05:40:28,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/bf06c24f7b704e0d8374b58186c581e8, entries=150, sequenceid=177, filesize=11.9 K 2024-11-13T05:40:28,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/408dcc5bb46b4075981dfc1797b35079 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/408dcc5bb46b4075981dfc1797b35079 2024-11-13T05:40:28,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/408dcc5bb46b4075981dfc1797b35079, entries=150, sequenceid=177, filesize=11.9 K 2024-11-13T05:40:28,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 2aab469b292723a934ce524f70b46f92 in 1094ms, sequenceid=177, compaction requested=true 2024-11-13T05:40:28,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:28,585 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:28,585 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:28,587 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:28,587 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/A is initiating minor compaction (all files) 2024-11-13T05:40:28,587 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/A in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:28,587 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/d8d8e390314a47d8b3a5003e561ce355, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/5eb40525177d405babf65bae69e59028, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/a862f04c3bfa4ec6b16639b52f721c74] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=108.3 K 2024-11-13T05:40:28,587 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:28,587 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/d8d8e390314a47d8b3a5003e561ce355, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/5eb40525177d405babf65bae69e59028, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/a862f04c3bfa4ec6b16639b52f721c74] 2024-11-13T05:40:28,588 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:28,588 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/B is initiating minor compaction (all files) 2024-11-13T05:40:28,588 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/B in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:28,588 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/6f96749bdd3c47bdafdd36fe2148b43c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/c1d624e35cf64d2aad4d7f29e6408fad, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/bf06c24f7b704e0d8374b58186c581e8] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=35.9 K 2024-11-13T05:40:28,591 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8d8e390314a47d8b3a5003e561ce355, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731476424078 2024-11-13T05:40:28,591 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f96749bdd3c47bdafdd36fe2148b43c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731476424078 2024-11-13T05:40:28,591 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting c1d624e35cf64d2aad4d7f29e6408fad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731476425143 2024-11-13T05:40:28,591 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5eb40525177d405babf65bae69e59028, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731476425143 2024-11-13T05:40:28,592 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting bf06c24f7b704e0d8374b58186c581e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731476427372 2024-11-13T05:40:28,592 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting a862f04c3bfa4ec6b16639b52f721c74, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731476427312 2024-11-13T05:40:28,606 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:28,610 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#B#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:28,611 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/b290e0f7389f4875883717721c12c6e9 is 50, key is test_row_0/B:col10/1731476427488/Put/seqid=0 2024-11-13T05:40:28,611 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411134f804f5de3f946dfad6795e3ec5deb28_2aab469b292723a934ce524f70b46f92 store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:28,614 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411134f804f5de3f946dfad6795e3ec5deb28_2aab469b292723a934ce524f70b46f92, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:28,614 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134f804f5de3f946dfad6795e3ec5deb28_2aab469b292723a934ce524f70b46f92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:28,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742002_1178 (size=12561) 2024-11-13T05:40:28,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742003_1179 (size=4469) 2024-11-13T05:40:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:28,718 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:28,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T05:40:28,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136e5f4a853cf64647bfae1a1984346775_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476427571/Put/seqid=0 2024-11-13T05:40:28,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742004_1180 (size=14794) 2024-11-13T05:40:28,769 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:28,775 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136e5f4a853cf64647bfae1a1984346775_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136e5f4a853cf64647bfae1a1984346775_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:28,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476488769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:28,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476488770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:28,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:28,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:28,779 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/4d92c37665e8470e95a7a959348994ae, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:28,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/4d92c37665e8470e95a7a959348994ae is 175, key is test_row_0/A:col10/1731476427571/Put/seqid=0 2024-11-13T05:40:28,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742005_1181 (size=39749) 2024-11-13T05:40:28,793 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/4d92c37665e8470e95a7a959348994ae 2024-11-13T05:40:28,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/2a590f2441bf41cc87d2c4659faac664 is 50, key is test_row_0/B:col10/1731476427571/Put/seqid=0 2024-11-13T05:40:28,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742006_1182 (size=12151) 2024-11-13T05:40:28,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/2a590f2441bf41cc87d2c4659faac664 2024-11-13T05:40:28,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/698293e53aa748bea999e98bdff51744 is 50, key is test_row_0/C:col10/1731476427571/Put/seqid=0 2024-11-13T05:40:28,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476488886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:28,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476488886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:28,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742007_1183 (size=12151) 2024-11-13T05:40:29,030 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/b290e0f7389f4875883717721c12c6e9 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/b290e0f7389f4875883717721c12c6e9 2024-11-13T05:40:29,036 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/B of 2aab469b292723a934ce524f70b46f92 into b290e0f7389f4875883717721c12c6e9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:29,036 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:29,036 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/B, priority=13, startTime=1731476428585; duration=0sec 2024-11-13T05:40:29,036 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:29,036 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:B 2024-11-13T05:40:29,036 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:29,037 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#A#compaction#150 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:29,038 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/afcad549be7348eab5ec9a7aa1043c53 is 175, key is test_row_0/A:col10/1731476427488/Put/seqid=0 2024-11-13T05:40:29,040 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:29,040 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/C is initiating minor compaction (all files) 2024-11-13T05:40:29,040 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/C in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:29,040 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d9a0d366460047b4bf4124127ce77167, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/91d2894a093c4c8090a15d528b568758, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/408dcc5bb46b4075981dfc1797b35079] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=35.9 K 2024-11-13T05:40:29,040 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a0d366460047b4bf4124127ce77167, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731476424078 2024-11-13T05:40:29,041 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 91d2894a093c4c8090a15d528b568758, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731476425143 2024-11-13T05:40:29,041 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 408dcc5bb46b4075981dfc1797b35079, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731476427372 2024-11-13T05:40:29,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742008_1184 (size=31515) 2024-11-13T05:40:29,050 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/afcad549be7348eab5ec9a7aa1043c53 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/afcad549be7348eab5ec9a7aa1043c53 2024-11-13T05:40:29,051 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#C#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:29,052 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/b4fa2218225e433a8ee2bad8662a9408 is 50, key is test_row_0/C:col10/1731476427488/Put/seqid=0 2024-11-13T05:40:29,057 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/A of 2aab469b292723a934ce524f70b46f92 into afcad549be7348eab5ec9a7aa1043c53(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:29,058 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:29,058 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/A, priority=13, startTime=1731476428584; duration=0sec 2024-11-13T05:40:29,058 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:29,058 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:A 2024-11-13T05:40:29,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742009_1185 (size=12561) 2024-11-13T05:40:29,067 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/b4fa2218225e433a8ee2bad8662a9408 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/b4fa2218225e433a8ee2bad8662a9408 2024-11-13T05:40:29,077 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/C of 2aab469b292723a934ce524f70b46f92 into b4fa2218225e433a8ee2bad8662a9408(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:29,077 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:29,077 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/C, priority=13, startTime=1731476428585; duration=0sec 2024-11-13T05:40:29,077 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:29,078 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:C 2024-11-13T05:40:29,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476489097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:29,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476489097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:29,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/698293e53aa748bea999e98bdff51744 2024-11-13T05:40:29,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/4d92c37665e8470e95a7a959348994ae as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/4d92c37665e8470e95a7a959348994ae 2024-11-13T05:40:29,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/4d92c37665e8470e95a7a959348994ae, entries=200, sequenceid=198, filesize=38.8 K 2024-11-13T05:40:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/2a590f2441bf41cc87d2c4659faac664 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/2a590f2441bf41cc87d2c4659faac664 2024-11-13T05:40:29,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/2a590f2441bf41cc87d2c4659faac664, entries=150, sequenceid=198, filesize=11.9 K 2024-11-13T05:40:29,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/698293e53aa748bea999e98bdff51744 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/698293e53aa748bea999e98bdff51744 2024-11-13T05:40:29,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/698293e53aa748bea999e98bdff51744, entries=150, sequenceid=198, filesize=11.9 K 2024-11-13T05:40:29,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 2aab469b292723a934ce524f70b46f92 in 621ms, sequenceid=198, compaction requested=false 2024-11-13T05:40:29,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T05:40:29,396 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:29,397 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-13T05:40:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T05:40:29,399 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:29,399 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:29,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:29,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T05:40:29,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139c817406ee1043ca8617997a17a89083_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476429409/Put/seqid=0 2024-11-13T05:40:29,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742010_1186 (size=17284) 2024-11-13T05:40:29,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476489499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:29,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476489502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T05:40:29,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T05:40:29,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:29,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:29,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:29,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:29,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:29,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476489606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:29,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476489617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:29,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T05:40:29,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:29,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:29,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:29,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:29,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T05:40:29,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476489816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:29,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476489827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:29,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T05:40:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:29,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:29,879 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:29,883 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139c817406ee1043ca8617997a17a89083_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139c817406ee1043ca8617997a17a89083_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:29,884 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/437cd020ff9a4c34a9fc08108c071d18, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:29,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/437cd020ff9a4c34a9fc08108c071d18 is 175, key is test_row_0/A:col10/1731476429409/Put/seqid=0 2024-11-13T05:40:29,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742011_1187 (size=48389) 2024-11-13T05:40:30,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T05:40:30,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:30,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:30,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:30,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:30,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T05:40:30,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476490126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:30,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:30,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:30,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:30,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476490137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:30,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T05:40:30,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:30,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:30,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:30,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:30,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:30,290 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/437cd020ff9a4c34a9fc08108c071d18 2024-11-13T05:40:30,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/d94535be41e54a4aae59c626daab0fa8 is 50, key is test_row_0/B:col10/1731476429409/Put/seqid=0 2024-11-13T05:40:30,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T05:40:30,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:30,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:30,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742012_1188 (size=12151) 2024-11-13T05:40:30,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/d94535be41e54a4aae59c626daab0fa8 2024-11-13T05:40:30,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/95f11b08a9f24c489d01fd16c533dd49 is 50, key is test_row_0/C:col10/1731476429409/Put/seqid=0 2024-11-13T05:40:30,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742013_1189 (size=12151) 2024-11-13T05:40:30,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/95f11b08a9f24c489d01fd16c533dd49 2024-11-13T05:40:30,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/437cd020ff9a4c34a9fc08108c071d18 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/437cd020ff9a4c34a9fc08108c071d18 2024-11-13T05:40:30,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/437cd020ff9a4c34a9fc08108c071d18, entries=250, sequenceid=218, filesize=47.3 K 2024-11-13T05:40:30,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/d94535be41e54a4aae59c626daab0fa8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/d94535be41e54a4aae59c626daab0fa8 2024-11-13T05:40:30,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/d94535be41e54a4aae59c626daab0fa8, entries=150, sequenceid=218, filesize=11.9 K 2024-11-13T05:40:30,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/95f11b08a9f24c489d01fd16c533dd49 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/95f11b08a9f24c489d01fd16c533dd49 2024-11-13T05:40:30,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/95f11b08a9f24c489d01fd16c533dd49, entries=150, sequenceid=218, filesize=11.9 K 2024-11-13T05:40:30,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 2aab469b292723a934ce524f70b46f92 in 995ms, sequenceid=218, compaction requested=true 2024-11-13T05:40:30,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:30,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:30,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:30,405 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:30,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:30,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:30,405 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:30,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:30,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:30,407 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:30,407 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/B is initiating minor compaction (all files) 2024-11-13T05:40:30,407 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/B in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:30,407 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/b290e0f7389f4875883717721c12c6e9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/2a590f2441bf41cc87d2c4659faac664, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/d94535be41e54a4aae59c626daab0fa8] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=36.0 K 2024-11-13T05:40:30,408 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:30,408 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/A is initiating minor compaction (all files) 2024-11-13T05:40:30,408 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/A in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:30,408 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/afcad549be7348eab5ec9a7aa1043c53, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/4d92c37665e8470e95a7a959348994ae, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/437cd020ff9a4c34a9fc08108c071d18] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=116.8 K 2024-11-13T05:40:30,408 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:30,408 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/afcad549be7348eab5ec9a7aa1043c53, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/4d92c37665e8470e95a7a959348994ae, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/437cd020ff9a4c34a9fc08108c071d18] 2024-11-13T05:40:30,408 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting b290e0f7389f4875883717721c12c6e9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731476427372 2024-11-13T05:40:30,409 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting afcad549be7348eab5ec9a7aa1043c53, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731476427372 2024-11-13T05:40:30,410 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a590f2441bf41cc87d2c4659faac664, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731476427556 2024-11-13T05:40:30,410 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d92c37665e8470e95a7a959348994ae, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731476427556 2024-11-13T05:40:30,410 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting d94535be41e54a4aae59c626daab0fa8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731476428742 2024-11-13T05:40:30,410 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 437cd020ff9a4c34a9fc08108c071d18, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731476428742 2024-11-13T05:40:30,435 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#B#compaction#159 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:30,436 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/c6fefa83ffe04b0bba450c7ba1674e78 is 50, key is test_row_0/B:col10/1731476429409/Put/seqid=0 2024-11-13T05:40:30,447 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:30,456 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113f1bd4df719eb46c28ff11f56ebeb7dad_2aab469b292723a934ce524f70b46f92 store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:30,459 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113f1bd4df719eb46c28ff11f56ebeb7dad_2aab469b292723a934ce524f70b46f92, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:30,459 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f1bd4df719eb46c28ff11f56ebeb7dad_2aab469b292723a934ce524f70b46f92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:30,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T05:40:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:30,471 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-13T05:40:30,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742014_1190 (size=12663) 2024-11-13T05:40:30,491 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/c6fefa83ffe04b0bba450c7ba1674e78 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/c6fefa83ffe04b0bba450c7ba1674e78 2024-11-13T05:40:30,499 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/B of 2aab469b292723a934ce524f70b46f92 into c6fefa83ffe04b0bba450c7ba1674e78(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:30,499 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:30,499 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/B, priority=13, startTime=1731476430405; duration=0sec 2024-11-13T05:40:30,499 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:30,499 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:B 2024-11-13T05:40:30,499 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:30,500 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:30,500 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/C is initiating minor compaction (all files) 2024-11-13T05:40:30,500 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/C in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:30,501 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/b4fa2218225e433a8ee2bad8662a9408, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/698293e53aa748bea999e98bdff51744, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/95f11b08a9f24c489d01fd16c533dd49] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=36.0 K 2024-11-13T05:40:30,501 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting b4fa2218225e433a8ee2bad8662a9408, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731476427372 2024-11-13T05:40:30,503 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 698293e53aa748bea999e98bdff51744, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731476427556 2024-11-13T05:40:30,504 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 95f11b08a9f24c489d01fd16c533dd49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731476428742 2024-11-13T05:40:30,519 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#C#compaction#161 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:30,520 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/263b6fcfcd2e4391be9929c2f732a915 is 50, key is test_row_0/C:col10/1731476429409/Put/seqid=0 2024-11-13T05:40:30,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742015_1191 (size=4469) 2024-11-13T05:40:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T05:40:30,556 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#A#compaction#160 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:30,557 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/2812b12bd3f14929bd393c21fd0e117d is 175, key is test_row_0/A:col10/1731476429409/Put/seqid=0 2024-11-13T05:40:30,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111391c887dd36ee4007bb98233ab16b58c3_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476429498/Put/seqid=0 2024-11-13T05:40:30,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742016_1192 (size=12663) 2024-11-13T05:40:30,582 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/263b6fcfcd2e4391be9929c2f732a915 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/263b6fcfcd2e4391be9929c2f732a915 2024-11-13T05:40:30,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742017_1193 (size=31617) 2024-11-13T05:40:30,594 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/C of 2aab469b292723a934ce524f70b46f92 into 263b6fcfcd2e4391be9929c2f732a915(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:30,594 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:30,594 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/C, priority=13, startTime=1731476430405; duration=0sec 2024-11-13T05:40:30,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742018_1194 (size=12304) 2024-11-13T05:40:30,594 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:30,595 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:C 2024-11-13T05:40:30,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:30,601 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/2812b12bd3f14929bd393c21fd0e117d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/2812b12bd3f14929bd393c21fd0e117d 2024-11-13T05:40:30,604 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111391c887dd36ee4007bb98233ab16b58c3_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111391c887dd36ee4007bb98233ab16b58c3_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:30,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/cc0bb3abac6942558a57921b3fd1aab9, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:30,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/cc0bb3abac6942558a57921b3fd1aab9 is 175, key is test_row_0/A:col10/1731476429498/Put/seqid=0 2024-11-13T05:40:30,613 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/A of 2aab469b292723a934ce524f70b46f92 into 2812b12bd3f14929bd393c21fd0e117d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:30,613 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:30,613 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/A, priority=13, startTime=1731476430405; duration=0sec 2024-11-13T05:40:30,613 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:30,614 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:A 2024-11-13T05:40:30,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742019_1195 (size=31105) 2024-11-13T05:40:30,636 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/cc0bb3abac6942558a57921b3fd1aab9 2024-11-13T05:40:30,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/4d3db8f1b206489780cb5c530ef39879 is 50, key is test_row_0/B:col10/1731476429498/Put/seqid=0 2024-11-13T05:40:30,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742020_1196 (size=12151) 2024-11-13T05:40:30,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476490722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:30,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476490723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:30,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476490836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:30,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476490836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:31,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476491046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:31,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476491046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:31,086 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/4d3db8f1b206489780cb5c530ef39879 2024-11-13T05:40:31,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/89faf4fbd7d04b2dadf0d1d51ca93a7f is 50, key is test_row_0/C:col10/1731476429498/Put/seqid=0 2024-11-13T05:40:31,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742021_1197 (size=12151) 2024-11-13T05:40:31,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476491357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:31,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476491357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:31,498 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/89faf4fbd7d04b2dadf0d1d51ca93a7f 2024-11-13T05:40:31,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/cc0bb3abac6942558a57921b3fd1aab9 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/cc0bb3abac6942558a57921b3fd1aab9 2024-11-13T05:40:31,511 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/cc0bb3abac6942558a57921b3fd1aab9, entries=150, sequenceid=237, filesize=30.4 K 2024-11-13T05:40:31,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/4d3db8f1b206489780cb5c530ef39879 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/4d3db8f1b206489780cb5c530ef39879 2024-11-13T05:40:31,519 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/4d3db8f1b206489780cb5c530ef39879, entries=150, sequenceid=237, filesize=11.9 K 2024-11-13T05:40:31,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/89faf4fbd7d04b2dadf0d1d51ca93a7f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/89faf4fbd7d04b2dadf0d1d51ca93a7f 2024-11-13T05:40:31,536 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/89faf4fbd7d04b2dadf0d1d51ca93a7f, entries=150, sequenceid=237, filesize=11.9 K 2024-11-13T05:40:31,537 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 2aab469b292723a934ce524f70b46f92 in 1066ms, sequenceid=237, compaction requested=false 2024-11-13T05:40:31,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:31,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:31,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-13T05:40:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-13T05:40:31,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-13T05:40:31,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1390 sec 2024-11-13T05:40:31,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.1450 sec 2024-11-13T05:40:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T05:40:31,567 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:31,567 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-13T05:40:31,569 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T05:40:31,570 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:31,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T05:40:31,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T05:40:31,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:31,726 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T05:40:31,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130bb7c5c8b7054967bb1143b9bf812c91_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476430703/Put/seqid=0 2024-11-13T05:40:31,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742022_1198 (size=12304) 2024-11-13T05:40:31,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:31,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T05:40:31,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:31,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476491923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:31,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:31,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:31,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476491924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:31,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:31,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:31,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:32,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:32,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476492036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:32,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476492036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:32,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:32,194 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130bb7c5c8b7054967bb1143b9bf812c91_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130bb7c5c8b7054967bb1143b9bf812c91_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:32,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/9adf4d0c9d8e4d95827865e832c02999, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:32,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/9adf4d0c9d8e4d95827865e832c02999 is 175, key is test_row_0/A:col10/1731476430703/Put/seqid=0 2024-11-13T05:40:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T05:40:32,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742023_1199 (size=31105) 2024-11-13T05:40:32,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:32,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476492246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:32,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:32,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476492246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:32,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:32,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1731476492287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:32,288 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in 2aab469b292723a934ce524f70b46f92 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:32,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:32,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:32,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43658 deadline: 1731476492306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:32,310 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_0' in 2aab469b292723a934ce524f70b46f92 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:32,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:32,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1731476492326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:32,332 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_0' in 2aab469b292723a934ce524f70b46f92 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:32,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476492556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:32,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476492556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:32,625 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/9adf4d0c9d8e4d95827865e832c02999 2024-11-13T05:40:32,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/0da37b7b23ba4a08ac533cd2da749f91 is 50, key is test_row_0/B:col10/1731476430703/Put/seqid=0 2024-11-13T05:40:32,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742024_1200 (size=12151) 2024-11-13T05:40:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T05:40:33,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476493067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:33,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476493067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:33,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:33,080 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/0da37b7b23ba4a08ac533cd2da749f91 2024-11-13T05:40:33,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/8694968fb5684b5ea34e6ae4653da063 is 50, key is test_row_0/C:col10/1731476430703/Put/seqid=0 2024-11-13T05:40:33,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742025_1201 (size=12151) 2024-11-13T05:40:33,116 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/8694968fb5684b5ea34e6ae4653da063 2024-11-13T05:40:33,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/9adf4d0c9d8e4d95827865e832c02999 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/9adf4d0c9d8e4d95827865e832c02999 2024-11-13T05:40:33,130 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/9adf4d0c9d8e4d95827865e832c02999, entries=150, sequenceid=257, filesize=30.4 K 2024-11-13T05:40:33,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/0da37b7b23ba4a08ac533cd2da749f91 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/0da37b7b23ba4a08ac533cd2da749f91 2024-11-13T05:40:33,139 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/0da37b7b23ba4a08ac533cd2da749f91, entries=150, sequenceid=257, filesize=11.9 K 2024-11-13T05:40:33,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/8694968fb5684b5ea34e6ae4653da063 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/8694968fb5684b5ea34e6ae4653da063 2024-11-13T05:40:33,145 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/8694968fb5684b5ea34e6ae4653da063, entries=150, sequenceid=257, filesize=11.9 K 2024-11-13T05:40:33,146 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 2aab469b292723a934ce524f70b46f92 in 1421ms, sequenceid=257, compaction requested=true 2024-11-13T05:40:33,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:33,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:33,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-13T05:40:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-13T05:40:33,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-13T05:40:33,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5780 sec 2024-11-13T05:40:33,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.5950 sec 2024-11-13T05:40:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T05:40:33,717 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:33,717 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-13T05:40:33,719 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T05:40:33,720 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:33,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T05:40:33,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:40:33,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:33,873 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-13T05:40:33,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134cf7c045334b4de89d46d3dfb0137305_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476431921/Put/seqid=0 2024-11-13T05:40:33,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742026_1202 (size=12454) 2024-11-13T05:40:33,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:33,942 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134cf7c045334b4de89d46d3dfb0137305_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134cf7c045334b4de89d46d3dfb0137305_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:33,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/743ee8f166344ec48751b7b8732150d8, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:33,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/743ee8f166344ec48751b7b8732150d8 is 175, key is test_row_0/A:col10/1731476431921/Put/seqid=0 2024-11-13T05:40:33,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742027_1203 (size=31255) 2024-11-13T05:40:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T05:40:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:34,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:34,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476494130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:34,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476494131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:34,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476494236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:34,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476494246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:34,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T05:40:34,353 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/743ee8f166344ec48751b7b8732150d8 2024-11-13T05:40:34,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/ba64f62674d8407590170facbfe01cc4 is 50, key is test_row_0/B:col10/1731476431921/Put/seqid=0 2024-11-13T05:40:34,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742028_1204 (size=12301) 2024-11-13T05:40:34,375 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/ba64f62674d8407590170facbfe01cc4 2024-11-13T05:40:34,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/1db5e527cdbb49529f06bb2b3f708c85 is 50, key is test_row_0/C:col10/1731476431921/Put/seqid=0 2024-11-13T05:40:34,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742029_1205 (size=12301) 2024-11-13T05:40:34,398 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/1db5e527cdbb49529f06bb2b3f708c85 2024-11-13T05:40:34,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/743ee8f166344ec48751b7b8732150d8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/743ee8f166344ec48751b7b8732150d8 2024-11-13T05:40:34,419 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/743ee8f166344ec48751b7b8732150d8, entries=150, sequenceid=276, filesize=30.5 K 2024-11-13T05:40:34,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/ba64f62674d8407590170facbfe01cc4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/ba64f62674d8407590170facbfe01cc4 2024-11-13T05:40:34,430 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/ba64f62674d8407590170facbfe01cc4, entries=150, sequenceid=276, filesize=12.0 K 2024-11-13T05:40:34,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/1db5e527cdbb49529f06bb2b3f708c85 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1db5e527cdbb49529f06bb2b3f708c85 2024-11-13T05:40:34,441 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1db5e527cdbb49529f06bb2b3f708c85, entries=150, sequenceid=276, filesize=12.0 K 2024-11-13T05:40:34,442 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 2aab469b292723a934ce524f70b46f92 in 569ms, sequenceid=276, compaction requested=true 2024-11-13T05:40:34,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:34,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:34,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-13T05:40:34,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-13T05:40:34,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-13T05:40:34,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 724 msec 2024-11-13T05:40:34,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 731 msec 2024-11-13T05:40:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:34,451 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:34,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-13T05:40:34,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130c4a4a56922a4b089f3659cc51779168_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476434450/Put/seqid=0 2024-11-13T05:40:34,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476494496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:34,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:34,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476494502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:34,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:34,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742030_1206 (size=12454) 2024-11-13T05:40:34,518 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:34,523 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130c4a4a56922a4b089f3659cc51779168_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130c4a4a56922a4b089f3659cc51779168_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:34,524 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/d288593edd554a51a07141e8a3857d42, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:34,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/d288593edd554a51a07141e8a3857d42 is 175, key is test_row_0/A:col10/1731476434450/Put/seqid=0 2024-11-13T05:40:34,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742031_1207 (size=31255) 2024-11-13T05:40:34,533 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/d288593edd554a51a07141e8a3857d42 2024-11-13T05:40:34,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/a3b080542cb440299b5867fa59e57c8c is 50, key is test_row_0/B:col10/1731476434450/Put/seqid=0 2024-11-13T05:40:34,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742032_1208 (size=12301) 2024-11-13T05:40:34,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/a3b080542cb440299b5867fa59e57c8c 2024-11-13T05:40:34,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/ca0e6fe55b13458b87657353fb3bc2e3 is 50, key is test_row_0/C:col10/1731476434450/Put/seqid=0 2024-11-13T05:40:34,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742033_1209 (size=12301) 2024-11-13T05:40:34,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/ca0e6fe55b13458b87657353fb3bc2e3 2024-11-13T05:40:34,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/d288593edd554a51a07141e8a3857d42 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/d288593edd554a51a07141e8a3857d42 2024-11-13T05:40:34,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/d288593edd554a51a07141e8a3857d42, entries=150, sequenceid=295, filesize=30.5 K 2024-11-13T05:40:34,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/a3b080542cb440299b5867fa59e57c8c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/a3b080542cb440299b5867fa59e57c8c 2024-11-13T05:40:34,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/a3b080542cb440299b5867fa59e57c8c, entries=150, sequenceid=295, filesize=12.0 K 2024-11-13T05:40:34,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/ca0e6fe55b13458b87657353fb3bc2e3 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/ca0e6fe55b13458b87657353fb3bc2e3 2024-11-13T05:40:34,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/ca0e6fe55b13458b87657353fb3bc2e3, entries=150, sequenceid=295, filesize=12.0 K 2024-11-13T05:40:34,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 2aab469b292723a934ce524f70b46f92 in 156ms, sequenceid=295, compaction requested=true 2024-11-13T05:40:34,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:34,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:34,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:34,607 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T05:40:34,607 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T05:40:34,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:34,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:34,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:34,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:34,609 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61567 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T05:40:34,610 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/B is initiating minor compaction (all files) 2024-11-13T05:40:34,610 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/B in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:34,611 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/c6fefa83ffe04b0bba450c7ba1674e78, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/4d3db8f1b206489780cb5c530ef39879, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/0da37b7b23ba4a08ac533cd2da749f91, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/ba64f62674d8407590170facbfe01cc4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/a3b080542cb440299b5867fa59e57c8c] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=60.1 K 2024-11-13T05:40:34,611 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 156337 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T05:40:34,611 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/A is initiating minor compaction (all files) 2024-11-13T05:40:34,611 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/A in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:34,611 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/2812b12bd3f14929bd393c21fd0e117d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/cc0bb3abac6942558a57921b3fd1aab9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/9adf4d0c9d8e4d95827865e832c02999, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/743ee8f166344ec48751b7b8732150d8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/d288593edd554a51a07141e8a3857d42] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=152.7 K 2024-11-13T05:40:34,611 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:34,611 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/2812b12bd3f14929bd393c21fd0e117d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/cc0bb3abac6942558a57921b3fd1aab9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/9adf4d0c9d8e4d95827865e832c02999, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/743ee8f166344ec48751b7b8732150d8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/d288593edd554a51a07141e8a3857d42] 2024-11-13T05:40:34,613 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting c6fefa83ffe04b0bba450c7ba1674e78, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731476428742 2024-11-13T05:40:34,613 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2812b12bd3f14929bd393c21fd0e117d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731476428742 2024-11-13T05:40:34,613 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d3db8f1b206489780cb5c530ef39879, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731476429454 2024-11-13T05:40:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:34,613 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc0bb3abac6942558a57921b3fd1aab9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731476429454 2024-11-13T05:40:34,615 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:34,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T05:40:34,615 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9adf4d0c9d8e4d95827865e832c02999, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731476430703 2024-11-13T05:40:34,615 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 0da37b7b23ba4a08ac533cd2da749f91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731476430703 2024-11-13T05:40:34,616 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 743ee8f166344ec48751b7b8732150d8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731476431898 2024-11-13T05:40:34,616 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting ba64f62674d8407590170facbfe01cc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731476431898 2024-11-13T05:40:34,617 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting a3b080542cb440299b5867fa59e57c8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731476434117 2024-11-13T05:40:34,618 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting d288593edd554a51a07141e8a3857d42, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731476434117 2024-11-13T05:40:34,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111312a7653f47af405b9532da180ca7026e_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476434612/Put/seqid=0 2024-11-13T05:40:34,647 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:34,665 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#B#compaction#176 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:34,666 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/1e2a1551df074894badb15bdb4154712 is 50, key is test_row_0/B:col10/1731476434450/Put/seqid=0 2024-11-13T05:40:34,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476494673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:34,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:34,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476494674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:34,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:34,683 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411138747095d9420425f99096034820d1fa9_2aab469b292723a934ce524f70b46f92 store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:34,686 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411138747095d9420425f99096034820d1fa9_2aab469b292723a934ce524f70b46f92, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:34,686 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138747095d9420425f99096034820d1fa9_2aab469b292723a934ce524f70b46f92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:34,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742034_1210 (size=12454) 2024-11-13T05:40:34,694 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:34,700 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111312a7653f47af405b9532da180ca7026e_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111312a7653f47af405b9532da180ca7026e_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:34,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742035_1211 (size=12983) 2024-11-13T05:40:34,701 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/b6c7f0725afd43cfb7a9ac8890208639, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:34,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/b6c7f0725afd43cfb7a9ac8890208639 is 175, key is test_row_0/A:col10/1731476434612/Put/seqid=0 2024-11-13T05:40:34,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742037_1213 (size=31255) 2024-11-13T05:40:34,717 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/b6c7f0725afd43cfb7a9ac8890208639 2024-11-13T05:40:34,722 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/1e2a1551df074894badb15bdb4154712 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/1e2a1551df074894badb15bdb4154712 2024-11-13T05:40:34,728 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 2aab469b292723a934ce524f70b46f92/B of 2aab469b292723a934ce524f70b46f92 into 1e2a1551df074894badb15bdb4154712(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:34,728 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:34,728 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/B, priority=11, startTime=1731476434607; duration=0sec 2024-11-13T05:40:34,728 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:34,728 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:B 2024-11-13T05:40:34,728 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T05:40:34,730 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61567 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T05:40:34,730 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/C is initiating minor compaction (all files) 2024-11-13T05:40:34,730 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/C in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:34,730 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/263b6fcfcd2e4391be9929c2f732a915, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/89faf4fbd7d04b2dadf0d1d51ca93a7f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/8694968fb5684b5ea34e6ae4653da063, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1db5e527cdbb49529f06bb2b3f708c85, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/ca0e6fe55b13458b87657353fb3bc2e3] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=60.1 K 2024-11-13T05:40:34,731 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 263b6fcfcd2e4391be9929c2f732a915, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731476428742 2024-11-13T05:40:34,731 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 89faf4fbd7d04b2dadf0d1d51ca93a7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731476429454 2024-11-13T05:40:34,731 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 8694968fb5684b5ea34e6ae4653da063, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731476430703 2024-11-13T05:40:34,732 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 1db5e527cdbb49529f06bb2b3f708c85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731476431898 2024-11-13T05:40:34,732 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting ca0e6fe55b13458b87657353fb3bc2e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731476434117 2024-11-13T05:40:34,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742036_1212 (size=4469) 2024-11-13T05:40:34,736 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#A#compaction#175 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:34,737 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/5b814f1467654ef68a8d6f50545486a0 is 175, key is test_row_0/A:col10/1731476434450/Put/seqid=0 2024-11-13T05:40:34,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/38749730addf4269bf3b0732da446f36 is 50, key is test_row_0/B:col10/1731476434612/Put/seqid=0 2024-11-13T05:40:34,766 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#C#compaction#178 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:34,766 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/c69e96e2937f48709a4158cd10b98151 is 50, key is test_row_0/C:col10/1731476434450/Put/seqid=0 2024-11-13T05:40:34,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476494786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:34,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476494786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:34,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742038_1214 (size=31937) 2024-11-13T05:40:34,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742039_1215 (size=12301) 2024-11-13T05:40:34,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742040_1216 (size=12983) 2024-11-13T05:40:34,814 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/c69e96e2937f48709a4158cd10b98151 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/c69e96e2937f48709a4158cd10b98151 2024-11-13T05:40:34,819 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 2aab469b292723a934ce524f70b46f92/C of 2aab469b292723a934ce524f70b46f92 into c69e96e2937f48709a4158cd10b98151(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:34,819 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:34,819 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/C, priority=11, startTime=1731476434608; duration=0sec 2024-11-13T05:40:34,820 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:34,820 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:C 2024-11-13T05:40:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T05:40:34,856 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:34,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-13T05:40:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T05:40:34,858 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:34,859 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:34,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T05:40:34,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476494996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:34,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476494996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:35,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:35,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T05:40:35,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:35,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:35,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:35,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T05:40:35,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:35,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:35,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:35,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T05:40:35,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/38749730addf4269bf3b0732da446f36 2024-11-13T05:40:35,204 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/5b814f1467654ef68a8d6f50545486a0 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/5b814f1467654ef68a8d6f50545486a0 2024-11-13T05:40:35,215 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 2aab469b292723a934ce524f70b46f92/A of 2aab469b292723a934ce524f70b46f92 into 5b814f1467654ef68a8d6f50545486a0(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:35,215 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:35,215 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/A, priority=11, startTime=1731476434607; duration=0sec 2024-11-13T05:40:35,216 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:35,216 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:A 2024-11-13T05:40:35,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/1ec585b381034581979865d4e6fc18ae is 50, key is test_row_0/C:col10/1731476434612/Put/seqid=0 2024-11-13T05:40:35,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742041_1217 (size=12301) 2024-11-13T05:40:35,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/1ec585b381034581979865d4e6fc18ae 2024-11-13T05:40:35,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/b6c7f0725afd43cfb7a9ac8890208639 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b6c7f0725afd43cfb7a9ac8890208639 2024-11-13T05:40:35,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b6c7f0725afd43cfb7a9ac8890208639, entries=150, sequenceid=314, filesize=30.5 K 2024-11-13T05:40:35,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/38749730addf4269bf3b0732da446f36 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/38749730addf4269bf3b0732da446f36 2024-11-13T05:40:35,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/38749730addf4269bf3b0732da446f36, entries=150, sequenceid=314, filesize=12.0 K 2024-11-13T05:40:35,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/1ec585b381034581979865d4e6fc18ae as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1ec585b381034581979865d4e6fc18ae 2024-11-13T05:40:35,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1ec585b381034581979865d4e6fc18ae, entries=150, sequenceid=314, filesize=12.0 K 2024-11-13T05:40:35,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 2aab469b292723a934ce524f70b46f92 in 636ms, sequenceid=314, compaction requested=false 2024-11-13T05:40:35,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:35,307 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:35,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T05:40:35,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T05:40:35,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:35,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:35,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:35,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111341bb967bd0e64f26952d6e1d7590a6d1_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476434672/Put/seqid=0 2024-11-13T05:40:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742042_1218 (size=14994) 2024-11-13T05:40:35,338 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:35,343 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111341bb967bd0e64f26952d6e1d7590a6d1_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111341bb967bd0e64f26952d6e1d7590a6d1_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:35,344 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/ccfa878a1f9b47bcaf46753e91fe976f, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:35,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/ccfa878a1f9b47bcaf46753e91fe976f is 175, key is test_row_0/A:col10/1731476434672/Put/seqid=0 2024-11-13T05:40:35,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476495352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:35,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742043_1219 (size=39949) 2024-11-13T05:40:35,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:35,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476495353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:35,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:35,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:35,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476495466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:35,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476495466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:35,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T05:40:35,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:35,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:35,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:35,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T05:40:35,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T05:40:35,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:35,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:35,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:35,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476495676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:35,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476495676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:35,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:35,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:35,761 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/ccfa878a1f9b47bcaf46753e91fe976f 2024-11-13T05:40:35,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T05:40:35,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:35,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:35,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:35,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/830ee678e7654e08bcb1ee6e69cf76d5 is 50, key is test_row_0/B:col10/1731476434672/Put/seqid=0 2024-11-13T05:40:35,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742044_1220 (size=12301) 2024-11-13T05:40:35,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T05:40:35,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:35,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:35,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:35,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:35,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476495987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:35,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476495987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T05:40:36,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T05:40:36,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:36,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:36,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:36,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:36,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:36,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/830ee678e7654e08bcb1ee6e69cf76d5 2024-11-13T05:40:36,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/59edf1c072494c9b9b6b5fd9b831b7c6 is 50, key is test_row_0/C:col10/1731476434672/Put/seqid=0 2024-11-13T05:40:36,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T05:40:36,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:36,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:36,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:36,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:36,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:36,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742045_1221 (size=12301) 2024-11-13T05:40:36,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T05:40:36,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:36,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:36,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:36,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:36,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476496497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:36,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476496497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:36,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T05:40:36,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:36,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:36,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:36,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:36,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:36,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/59edf1c072494c9b9b6b5fd9b831b7c6 2024-11-13T05:40:36,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/ccfa878a1f9b47bcaf46753e91fe976f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/ccfa878a1f9b47bcaf46753e91fe976f 2024-11-13T05:40:36,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/ccfa878a1f9b47bcaf46753e91fe976f, entries=200, sequenceid=335, filesize=39.0 K 2024-11-13T05:40:36,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/830ee678e7654e08bcb1ee6e69cf76d5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/830ee678e7654e08bcb1ee6e69cf76d5 2024-11-13T05:40:36,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/830ee678e7654e08bcb1ee6e69cf76d5, entries=150, sequenceid=335, filesize=12.0 K 2024-11-13T05:40:36,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/59edf1c072494c9b9b6b5fd9b831b7c6 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/59edf1c072494c9b9b6b5fd9b831b7c6 2024-11-13T05:40:36,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/59edf1c072494c9b9b6b5fd9b831b7c6, entries=150, sequenceid=335, filesize=12.0 K 2024-11-13T05:40:36,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 2aab469b292723a934ce524f70b46f92 in 1368ms, sequenceid=335, compaction requested=true 2024-11-13T05:40:36,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:36,675 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:36,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:36,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:36,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:36,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:36,676 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:36,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:36,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:36,677 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:36,677 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/A is initiating minor compaction (all files) 2024-11-13T05:40:36,677 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/A in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:36,677 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/5b814f1467654ef68a8d6f50545486a0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b6c7f0725afd43cfb7a9ac8890208639, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/ccfa878a1f9b47bcaf46753e91fe976f] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=100.7 K 2024-11-13T05:40:36,677 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:36,677 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/5b814f1467654ef68a8d6f50545486a0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b6c7f0725afd43cfb7a9ac8890208639, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/ccfa878a1f9b47bcaf46753e91fe976f] 2024-11-13T05:40:36,677 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b814f1467654ef68a8d6f50545486a0, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731476434117 2024-11-13T05:40:36,678 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6c7f0725afd43cfb7a9ac8890208639, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731476434481 2024-11-13T05:40:36,678 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:36,678 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/B is initiating minor compaction (all files) 2024-11-13T05:40:36,678 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/B in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:36,678 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/1e2a1551df074894badb15bdb4154712, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/38749730addf4269bf3b0732da446f36, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/830ee678e7654e08bcb1ee6e69cf76d5] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=36.7 K 2024-11-13T05:40:36,678 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e2a1551df074894badb15bdb4154712, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731476434117 2024-11-13T05:40:36,678 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccfa878a1f9b47bcaf46753e91fe976f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731476434652 2024-11-13T05:40:36,679 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 38749730addf4269bf3b0732da446f36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731476434481 2024-11-13T05:40:36,679 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 830ee678e7654e08bcb1ee6e69cf76d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731476434671 2024-11-13T05:40:36,686 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:36,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T05:40:36,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:36,697 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T05:40:36,699 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113378815a8c7174c7b97eddb652004544a_2aab469b292723a934ce524f70b46f92 store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:36,701 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113378815a8c7174c7b97eddb652004544a_2aab469b292723a934ce524f70b46f92, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:36,701 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113378815a8c7174c7b97eddb652004544a_2aab469b292723a934ce524f70b46f92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:36,707 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#B#compaction#184 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:36,707 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/d199d1f2878b4e8d8307bb1280b09f17 is 50, key is test_row_0/B:col10/1731476434672/Put/seqid=0 2024-11-13T05:40:36,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139d5a272fd42d430094505dc46696baaa_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476435350/Put/seqid=0 2024-11-13T05:40:36,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742046_1222 (size=4469) 2024-11-13T05:40:36,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742047_1223 (size=13085) 2024-11-13T05:40:36,755 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/d199d1f2878b4e8d8307bb1280b09f17 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/d199d1f2878b4e8d8307bb1280b09f17 2024-11-13T05:40:36,762 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/B of 2aab469b292723a934ce524f70b46f92 into d199d1f2878b4e8d8307bb1280b09f17(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:36,762 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:36,762 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/B, priority=13, startTime=1731476436676; duration=0sec 2024-11-13T05:40:36,762 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:36,762 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:B 2024-11-13T05:40:36,762 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:36,768 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:36,768 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/C is initiating minor compaction (all files) 2024-11-13T05:40:36,768 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/C in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:36,768 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/c69e96e2937f48709a4158cd10b98151, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1ec585b381034581979865d4e6fc18ae, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/59edf1c072494c9b9b6b5fd9b831b7c6] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=36.7 K 2024-11-13T05:40:36,769 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting c69e96e2937f48709a4158cd10b98151, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731476434117 2024-11-13T05:40:36,769 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ec585b381034581979865d4e6fc18ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731476434481 2024-11-13T05:40:36,769 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 59edf1c072494c9b9b6b5fd9b831b7c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731476434671 2024-11-13T05:40:36,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742048_1224 (size=12454) 2024-11-13T05:40:36,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:36,787 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#C#compaction#186 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:36,788 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/f147c47e47cb47419f4f0ceaa676ebbe is 50, key is test_row_0/C:col10/1731476434672/Put/seqid=0 2024-11-13T05:40:36,789 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139d5a272fd42d430094505dc46696baaa_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139d5a272fd42d430094505dc46696baaa_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:36,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/cb21ec66a32245c7a86e77215da34831, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:36,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/cb21ec66a32245c7a86e77215da34831 is 175, key is test_row_0/A:col10/1731476435350/Put/seqid=0 2024-11-13T05:40:36,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742049_1225 (size=13085) 2024-11-13T05:40:36,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742050_1226 (size=31255) 2024-11-13T05:40:37,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T05:40:37,136 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#A#compaction#183 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:37,136 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/911d8e0f92a44bb0a94f277474d6fa41 is 175, key is test_row_0/A:col10/1731476434672/Put/seqid=0 2024-11-13T05:40:37,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742051_1227 (size=32039) 2024-11-13T05:40:37,216 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/cb21ec66a32245c7a86e77215da34831 2024-11-13T05:40:37,226 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/f147c47e47cb47419f4f0ceaa676ebbe as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/f147c47e47cb47419f4f0ceaa676ebbe 2024-11-13T05:40:37,237 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/C of 2aab469b292723a934ce524f70b46f92 into f147c47e47cb47419f4f0ceaa676ebbe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:37,237 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:37,237 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/C, priority=13, startTime=1731476436676; duration=0sec 2024-11-13T05:40:37,237 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:37,237 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:C 2024-11-13T05:40:37,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/348cacb1bf2a4a65b6f668ebffe13cb4 is 50, key is test_row_0/B:col10/1731476435350/Put/seqid=0 2024-11-13T05:40:37,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742052_1228 (size=12301) 2024-11-13T05:40:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:37,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. as already flushing 2024-11-13T05:40:37,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476497544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:37,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:37,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476497544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:37,599 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/911d8e0f92a44bb0a94f277474d6fa41 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/911d8e0f92a44bb0a94f277474d6fa41 2024-11-13T05:40:37,607 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/A of 2aab469b292723a934ce524f70b46f92 into 911d8e0f92a44bb0a94f277474d6fa41(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:37,607 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:37,607 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/A, priority=13, startTime=1731476436675; duration=0sec 2024-11-13T05:40:37,607 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:37,607 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:A 2024-11-13T05:40:37,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:37,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476497656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:37,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:37,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476497656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:37,686 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/348cacb1bf2a4a65b6f668ebffe13cb4 2024-11-13T05:40:37,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/293fe46b34874e3c85626715c22b5e75 is 50, key is test_row_0/C:col10/1731476435350/Put/seqid=0 2024-11-13T05:40:37,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742053_1229 (size=12301) 2024-11-13T05:40:37,712 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/293fe46b34874e3c85626715c22b5e75 2024-11-13T05:40:37,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/cb21ec66a32245c7a86e77215da34831 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/cb21ec66a32245c7a86e77215da34831 2024-11-13T05:40:37,724 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/cb21ec66a32245c7a86e77215da34831, entries=150, sequenceid=353, filesize=30.5 K 2024-11-13T05:40:37,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/348cacb1bf2a4a65b6f668ebffe13cb4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/348cacb1bf2a4a65b6f668ebffe13cb4 2024-11-13T05:40:37,730 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/348cacb1bf2a4a65b6f668ebffe13cb4, entries=150, sequenceid=353, filesize=12.0 K 2024-11-13T05:40:37,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/293fe46b34874e3c85626715c22b5e75 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/293fe46b34874e3c85626715c22b5e75 2024-11-13T05:40:37,740 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/293fe46b34874e3c85626715c22b5e75, entries=150, sequenceid=353, filesize=12.0 K 2024-11-13T05:40:37,742 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 2aab469b292723a934ce524f70b46f92 in 1045ms, sequenceid=353, compaction requested=false 2024-11-13T05:40:37,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:37,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:37,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-13T05:40:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-13T05:40:37,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-13T05:40:37,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8860 sec 2024-11-13T05:40:37,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.8940 sec 2024-11-13T05:40:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:37,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-13T05:40:37,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111331ac143ce21e4173aacb6ed526c76bf5_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476437868/Put/seqid=0 2024-11-13T05:40:37,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742054_1230 (size=14994) 2024-11-13T05:40:37,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476497939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:37,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476497939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:37,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:37,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:37,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:38,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476498046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:38,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476498056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:38,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476498256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:38,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:38,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476498266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:38,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:38,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:38,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:38,293 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:38,297 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111331ac143ce21e4173aacb6ed526c76bf5_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111331ac143ce21e4173aacb6ed526c76bf5_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:38,300 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/e4b61acd3dbc41b8a4c86671e6b3e6ce, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:38,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/e4b61acd3dbc41b8a4c86671e6b3e6ce is 175, key is test_row_0/A:col10/1731476437868/Put/seqid=0 2024-11-13T05:40:38,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742055_1231 (size=39949) 2024-11-13T05:40:38,391 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:40:38,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476498566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:38,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:38,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:38,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:38,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476498577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:38,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:38,717 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/e4b61acd3dbc41b8a4c86671e6b3e6ce 2024-11-13T05:40:38,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/9fe10d04946540e7a88860b5caab746f is 50, key is test_row_0/B:col10/1731476437868/Put/seqid=0 2024-11-13T05:40:38,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742056_1232 (size=12301) 2024-11-13T05:40:38,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/9fe10d04946540e7a88860b5caab746f 2024-11-13T05:40:38,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/d8ea5b0f311c40ab8f358e147d7d5a23 is 50, key is test_row_0/C:col10/1731476437868/Put/seqid=0 2024-11-13T05:40:38,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742057_1233 (size=12301) 2024-11-13T05:40:38,912 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-11-13T05:40:38,912 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:38,912 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:38,913 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:38,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:38,917 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-13T05:40:38,917 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:38,917 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:38,917 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:38,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:38,919 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-13T05:40:38,919 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:38,919 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:38,919 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:38,919 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:38,925 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-11-13T05:40:38,925 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:38,925 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:38,925 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:38,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T05:40:39,026 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:39,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:39,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1731476499087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43670 deadline: 1731476499087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:40:39,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/d8ea5b0f311c40ab8f358e147d7d5a23 2024-11-13T05:40:39,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/e4b61acd3dbc41b8a4c86671e6b3e6ce as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/e4b61acd3dbc41b8a4c86671e6b3e6ce 2024-11-13T05:40:39,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/e4b61acd3dbc41b8a4c86671e6b3e6ce, entries=200, sequenceid=375, filesize=39.0 K 2024-11-13T05:40:39,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/9fe10d04946540e7a88860b5caab746f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/9fe10d04946540e7a88860b5caab746f 2024-11-13T05:40:39,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/9fe10d04946540e7a88860b5caab746f, entries=150, sequenceid=375, filesize=12.0 K 2024-11-13T05:40:39,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/d8ea5b0f311c40ab8f358e147d7d5a23 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d8ea5b0f311c40ab8f358e147d7d5a23 2024-11-13T05:40:39,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d8ea5b0f311c40ab8f358e147d7d5a23, entries=150, sequenceid=375, filesize=12.0 K 2024-11-13T05:40:39,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 2aab469b292723a934ce524f70b46f92 in 1303ms, sequenceid=375, compaction requested=true 2024-11-13T05:40:39,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:39,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:39,172 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:39,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:39,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:39,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:39,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2aab469b292723a934ce524f70b46f92:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:39,172 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:39,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:39,173 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:39,173 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:39,173 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/A is initiating minor compaction (all files) 2024-11-13T05:40:39,173 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/B is initiating minor compaction (all files) 2024-11-13T05:40:39,173 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/A in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:39,173 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/B in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:39,173 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/911d8e0f92a44bb0a94f277474d6fa41, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/cb21ec66a32245c7a86e77215da34831, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/e4b61acd3dbc41b8a4c86671e6b3e6ce] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=100.8 K 2024-11-13T05:40:39,173 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/d199d1f2878b4e8d8307bb1280b09f17, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/348cacb1bf2a4a65b6f668ebffe13cb4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/9fe10d04946540e7a88860b5caab746f] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=36.8 K 2024-11-13T05:40:39,173 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:39,173 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/911d8e0f92a44bb0a94f277474d6fa41, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/cb21ec66a32245c7a86e77215da34831, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/e4b61acd3dbc41b8a4c86671e6b3e6ce] 2024-11-13T05:40:39,173 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting d199d1f2878b4e8d8307bb1280b09f17, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731476434671 2024-11-13T05:40:39,173 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 911d8e0f92a44bb0a94f277474d6fa41, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731476434671 2024-11-13T05:40:39,174 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 348cacb1bf2a4a65b6f668ebffe13cb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731476435332 2024-11-13T05:40:39,174 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb21ec66a32245c7a86e77215da34831, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731476435332 2024-11-13T05:40:39,174 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fe10d04946540e7a88860b5caab746f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731476437535 2024-11-13T05:40:39,174 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4b61acd3dbc41b8a4c86671e6b3e6ce, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731476437535 2024-11-13T05:40:39,183 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#B#compaction#192 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:39,183 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/368b35ebfe0c4e7595b26c2bb4a17f75 is 50, key is test_row_0/B:col10/1731476437868/Put/seqid=0 2024-11-13T05:40:39,185 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:39,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742058_1234 (size=13187) 2024-11-13T05:40:39,187 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411136dcb65becf2648af9ad918b48b5e7758_2aab469b292723a934ce524f70b46f92 store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:39,211 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411136dcb65becf2648af9ad918b48b5e7758_2aab469b292723a934ce524f70b46f92, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:39,211 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136dcb65becf2648af9ad918b48b5e7758_2aab469b292723a934ce524f70b46f92 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:39,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742059_1235 (size=4469) 2024-11-13T05:40:39,219 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#A#compaction#193 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:39,220 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/a8b4e0afc63048ac9c836965538dbee4 is 175, key is test_row_0/A:col10/1731476437868/Put/seqid=0 2024-11-13T05:40:39,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742060_1236 (size=32141) 2024-11-13T05:40:39,234 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/a8b4e0afc63048ac9c836965538dbee4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/a8b4e0afc63048ac9c836965538dbee4 2024-11-13T05:40:39,240 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/A of 2aab469b292723a934ce524f70b46f92 into a8b4e0afc63048ac9c836965538dbee4(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:39,240 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:39,241 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/A, priority=13, startTime=1731476439172; duration=0sec 2024-11-13T05:40:39,241 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:39,241 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:A 2024-11-13T05:40:39,241 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:39,243 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:39,243 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 2aab469b292723a934ce524f70b46f92/C is initiating minor compaction (all files) 2024-11-13T05:40:39,243 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2aab469b292723a934ce524f70b46f92/C in TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:39,243 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/f147c47e47cb47419f4f0ceaa676ebbe, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/293fe46b34874e3c85626715c22b5e75, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d8ea5b0f311c40ab8f358e147d7d5a23] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp, totalSize=36.8 K 2024-11-13T05:40:39,244 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting f147c47e47cb47419f4f0ceaa676ebbe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731476434671 2024-11-13T05:40:39,244 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 293fe46b34874e3c85626715c22b5e75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731476435332 2024-11-13T05:40:39,245 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8ea5b0f311c40ab8f358e147d7d5a23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731476437535 2024-11-13T05:40:39,267 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2aab469b292723a934ce524f70b46f92#C#compaction#194 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:39,268 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/1868e1120e284858b941cbe9cea1a9ab is 50, key is test_row_0/C:col10/1731476437868/Put/seqid=0 2024-11-13T05:40:39,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742061_1237 (size=13187) 2024-11-13T05:40:39,280 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/1868e1120e284858b941cbe9cea1a9ab as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1868e1120e284858b941cbe9cea1a9ab 2024-11-13T05:40:39,287 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/C of 2aab469b292723a934ce524f70b46f92 into 1868e1120e284858b941cbe9cea1a9ab(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:39,287 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:39,287 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/C, priority=13, startTime=1731476439172; duration=0sec 2024-11-13T05:40:39,287 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:39,287 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:C 2024-11-13T05:40:39,593 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/368b35ebfe0c4e7595b26c2bb4a17f75 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/368b35ebfe0c4e7595b26c2bb4a17f75 2024-11-13T05:40:39,601 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2aab469b292723a934ce524f70b46f92/B of 2aab469b292723a934ce524f70b46f92 into 368b35ebfe0c4e7595b26c2bb4a17f75(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:39,601 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:39,601 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., storeName=2aab469b292723a934ce524f70b46f92/B, priority=13, startTime=1731476439172; duration=0sec 2024-11-13T05:40:39,601 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:39,601 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2aab469b292723a934ce524f70b46f92:B 2024-11-13T05:40:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:40,126 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:40,126 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-11-13T05:40:40,126 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-13T05:40:40,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T05:40:40,126 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:40,127 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:40,127 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:40,127 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:40,127 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:40,127 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:40,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:40,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:40,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135ada5b74577e408083844a18cc8a1884_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476437906/Put/seqid=0 2024-11-13T05:40:40,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742062_1238 (size=12454) 2024-11-13T05:40:40,144 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:40,149 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135ada5b74577e408083844a18cc8a1884_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135ada5b74577e408083844a18cc8a1884_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:40,154 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/e0db031d03f84eb49dddcf2b3889e254, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:40,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/e0db031d03f84eb49dddcf2b3889e254 is 175, key is test_row_0/A:col10/1731476437906/Put/seqid=0 2024-11-13T05:40:40,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742063_1239 (size=31255) 2024-11-13T05:40:40,164 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/e0db031d03f84eb49dddcf2b3889e254 2024-11-13T05:40:40,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/645efa2cbfb24f9593cffd07545197bf is 50, key is test_row_0/B:col10/1731476437906/Put/seqid=0 2024-11-13T05:40:40,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742064_1240 (size=12301) 2024-11-13T05:40:40,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/645efa2cbfb24f9593cffd07545197bf 2024-11-13T05:40:40,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/38e8db17ade64a68a6c4c18d472dbad1 is 50, key is test_row_0/C:col10/1731476437906/Put/seqid=0 2024-11-13T05:40:40,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742065_1241 (size=12301) 2024-11-13T05:40:40,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/38e8db17ade64a68a6c4c18d472dbad1 2024-11-13T05:40:40,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/e0db031d03f84eb49dddcf2b3889e254 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/e0db031d03f84eb49dddcf2b3889e254 2024-11-13T05:40:40,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/e0db031d03f84eb49dddcf2b3889e254, entries=150, sequenceid=398, filesize=30.5 K 2024-11-13T05:40:40,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/645efa2cbfb24f9593cffd07545197bf as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/645efa2cbfb24f9593cffd07545197bf 2024-11-13T05:40:40,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/645efa2cbfb24f9593cffd07545197bf, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T05:40:40,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/38e8db17ade64a68a6c4c18d472dbad1 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/38e8db17ade64a68a6c4c18d472dbad1 2024-11-13T05:40:40,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/38e8db17ade64a68a6c4c18d472dbad1, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T05:40:40,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 2aab469b292723a934ce524f70b46f92 in 521ms, sequenceid=398, compaction requested=false 2024-11-13T05:40:40,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2aab469b292723a934ce524f70b46f92: 2024-11-13T05:40:42,355 INFO [Thread-655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-655. 2024-11-13T05:40:42,355 DEBUG [Thread-655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:42,355 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:42,355 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:42,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:42,389 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-13T05:40:42,389 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:42,389 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:42,389 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:42,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:42,399 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-13T05:40:42,399 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:40:42,399 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:42,399 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:42,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:42,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T05:40:42,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-13T05:40:42,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-13T05:40:42,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-13T05:40:42,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-13T05:40:42,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-13T05:40:42,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T05:40:42,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3001 2024-11-13T05:40:42,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2953 2024-11-13T05:40:42,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T05:40:42,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1355 2024-11-13T05:40:42,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4060 rows 2024-11-13T05:40:42,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1346 2024-11-13T05:40:42,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4037 rows 2024-11-13T05:40:42,400 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:40:42,400 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:40:42,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:42,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:42,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:40:42,401 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T05:40:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T05:40:42,404 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476442404"}]},"ts":"1731476442404"} 2024-11-13T05:40:42,405 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T05:40:42,405 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T05:40:42,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:40:42,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, UNASSIGN}] 2024-11-13T05:40:42,408 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, UNASSIGN 2024-11-13T05:40:42,408 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=48 updating hbase:meta row=2aab469b292723a934ce524f70b46f92, regionState=CLOSING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:42,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, UNASSIGN because future has completed 2024-11-13T05:40:42,410 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:40:42,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:40:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T05:40:42,562 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(122): Close 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:42,563 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:40:42,563 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1722): Closing 2aab469b292723a934ce524f70b46f92, disabling compactions & flushes 2024-11-13T05:40:42,563 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:42,563 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:42,563 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. after waiting 0 ms 2024-11-13T05:40:42,563 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:42,563 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(2902): Flushing 2aab469b292723a934ce524f70b46f92 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-13T05:40:42,570 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ea9afc720f634a4abbef289e7a70378e_2aab469b292723a934ce524f70b46f92 is 50, key is test_row_0/A:col10/1731476442397/Put/seqid=0 2024-11-13T05:40:42,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742066_1242 (size=9914) 2024-11-13T05:40:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T05:40:42,975 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:42,982 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ea9afc720f634a4abbef289e7a70378e_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ea9afc720f634a4abbef289e7a70378e_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:42,983 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/92d1f111f5ab48678228fb35e7a14e1f, store: [table=TestAcidGuarantees family=A region=2aab469b292723a934ce524f70b46f92] 2024-11-13T05:40:42,984 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/92d1f111f5ab48678228fb35e7a14e1f is 175, key is test_row_0/A:col10/1731476442397/Put/seqid=0 2024-11-13T05:40:42,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742067_1243 (size=22561) 2024-11-13T05:40:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T05:40:43,390 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/92d1f111f5ab48678228fb35e7a14e1f 2024-11-13T05:40:43,406 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/788c872ee2f0466690314857c84260ab is 50, key is test_row_0/B:col10/1731476442397/Put/seqid=0 2024-11-13T05:40:43,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742068_1244 (size=9857) 2024-11-13T05:40:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T05:40:43,812 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/788c872ee2f0466690314857c84260ab 2024-11-13T05:40:43,823 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/bd8c73353b204767bb0b1eb71af4a57d is 50, key is test_row_0/C:col10/1731476442397/Put/seqid=0 2024-11-13T05:40:43,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742069_1245 (size=9857) 2024-11-13T05:40:44,229 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/bd8c73353b204767bb0b1eb71af4a57d 2024-11-13T05:40:44,240 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/A/92d1f111f5ab48678228fb35e7a14e1f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/92d1f111f5ab48678228fb35e7a14e1f 2024-11-13T05:40:44,243 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/92d1f111f5ab48678228fb35e7a14e1f, entries=100, sequenceid=404, filesize=22.0 K 2024-11-13T05:40:44,244 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/B/788c872ee2f0466690314857c84260ab as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/788c872ee2f0466690314857c84260ab 2024-11-13T05:40:44,247 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/788c872ee2f0466690314857c84260ab, entries=100, sequenceid=404, filesize=9.6 K 2024-11-13T05:40:44,248 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/.tmp/C/bd8c73353b204767bb0b1eb71af4a57d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/bd8c73353b204767bb0b1eb71af4a57d 2024-11-13T05:40:44,251 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/bd8c73353b204767bb0b1eb71af4a57d, entries=100, sequenceid=404, filesize=9.6 K 2024-11-13T05:40:44,252 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 2aab469b292723a934ce524f70b46f92 in 1689ms, sequenceid=404, compaction requested=true 2024-11-13T05:40:44,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b12486a1f002458f9eeb758aa7cab94e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/9939e5ea1be84943a76a88d5547cd487, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/61ef8808520b43b1a4b3c98f6cff072d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/58c36ea653cc490888b001b3369fc314, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/afecfa1117d24e3987d24632fa045495, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/20e3ec5b8f8848e395659cf43a01e208, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b454374a60ec41eca51970911a6a5ff0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/6812399ba5494fb7a2c6ff0ad8386656, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/acd1743d3ee7499d983e8ea6709f04d7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/d8d8e390314a47d8b3a5003e561ce355, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/5eb40525177d405babf65bae69e59028, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/a862f04c3bfa4ec6b16639b52f721c74, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/afcad549be7348eab5ec9a7aa1043c53, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/4d92c37665e8470e95a7a959348994ae, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/437cd020ff9a4c34a9fc08108c071d18, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/2812b12bd3f14929bd393c21fd0e117d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/cc0bb3abac6942558a57921b3fd1aab9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/9adf4d0c9d8e4d95827865e832c02999, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/743ee8f166344ec48751b7b8732150d8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/5b814f1467654ef68a8d6f50545486a0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/d288593edd554a51a07141e8a3857d42, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b6c7f0725afd43cfb7a9ac8890208639, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/ccfa878a1f9b47bcaf46753e91fe976f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/911d8e0f92a44bb0a94f277474d6fa41, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/cb21ec66a32245c7a86e77215da34831, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/e4b61acd3dbc41b8a4c86671e6b3e6ce] to archive 2024-11-13T05:40:44,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:40:44,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b12486a1f002458f9eeb758aa7cab94e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b12486a1f002458f9eeb758aa7cab94e 2024-11-13T05:40:44,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/9939e5ea1be84943a76a88d5547cd487 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/9939e5ea1be84943a76a88d5547cd487 2024-11-13T05:40:44,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/61ef8808520b43b1a4b3c98f6cff072d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/61ef8808520b43b1a4b3c98f6cff072d 2024-11-13T05:40:44,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/58c36ea653cc490888b001b3369fc314 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/58c36ea653cc490888b001b3369fc314 2024-11-13T05:40:44,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/afecfa1117d24e3987d24632fa045495 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/afecfa1117d24e3987d24632fa045495 2024-11-13T05:40:44,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/20e3ec5b8f8848e395659cf43a01e208 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/20e3ec5b8f8848e395659cf43a01e208 2024-11-13T05:40:44,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b454374a60ec41eca51970911a6a5ff0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b454374a60ec41eca51970911a6a5ff0 2024-11-13T05:40:44,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/6812399ba5494fb7a2c6ff0ad8386656 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/6812399ba5494fb7a2c6ff0ad8386656 2024-11-13T05:40:44,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/acd1743d3ee7499d983e8ea6709f04d7 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/acd1743d3ee7499d983e8ea6709f04d7 2024-11-13T05:40:44,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/d8d8e390314a47d8b3a5003e561ce355 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/d8d8e390314a47d8b3a5003e561ce355 2024-11-13T05:40:44,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/5eb40525177d405babf65bae69e59028 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/5eb40525177d405babf65bae69e59028 2024-11-13T05:40:44,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/a862f04c3bfa4ec6b16639b52f721c74 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/a862f04c3bfa4ec6b16639b52f721c74 2024-11-13T05:40:44,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/afcad549be7348eab5ec9a7aa1043c53 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/afcad549be7348eab5ec9a7aa1043c53 2024-11-13T05:40:44,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/4d92c37665e8470e95a7a959348994ae to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/4d92c37665e8470e95a7a959348994ae 2024-11-13T05:40:44,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/437cd020ff9a4c34a9fc08108c071d18 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/437cd020ff9a4c34a9fc08108c071d18 2024-11-13T05:40:44,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/2812b12bd3f14929bd393c21fd0e117d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/2812b12bd3f14929bd393c21fd0e117d 2024-11-13T05:40:44,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/cc0bb3abac6942558a57921b3fd1aab9 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/cc0bb3abac6942558a57921b3fd1aab9 2024-11-13T05:40:44,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/9adf4d0c9d8e4d95827865e832c02999 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/9adf4d0c9d8e4d95827865e832c02999 2024-11-13T05:40:44,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/743ee8f166344ec48751b7b8732150d8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/743ee8f166344ec48751b7b8732150d8 2024-11-13T05:40:44,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/5b814f1467654ef68a8d6f50545486a0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/5b814f1467654ef68a8d6f50545486a0 2024-11-13T05:40:44,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/d288593edd554a51a07141e8a3857d42 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/d288593edd554a51a07141e8a3857d42 2024-11-13T05:40:44,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b6c7f0725afd43cfb7a9ac8890208639 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/b6c7f0725afd43cfb7a9ac8890208639 2024-11-13T05:40:44,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/ccfa878a1f9b47bcaf46753e91fe976f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/ccfa878a1f9b47bcaf46753e91fe976f 2024-11-13T05:40:44,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/911d8e0f92a44bb0a94f277474d6fa41 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/911d8e0f92a44bb0a94f277474d6fa41 2024-11-13T05:40:44,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/cb21ec66a32245c7a86e77215da34831 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/cb21ec66a32245c7a86e77215da34831 2024-11-13T05:40:44,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/e4b61acd3dbc41b8a4c86671e6b3e6ce to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/e4b61acd3dbc41b8a4c86671e6b3e6ce 2024-11-13T05:40:44,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/74855d58134549389a45ac046294f2f6, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/3e723dd5e41f42f3be5c0ffcd39fe5ea, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/8c500af0fe5f4e1db96427f5444a1f91, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/2715aa1684664b6b958aa4d021724a67, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/3f395de68f284337b1db66ac44195a17, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/e186c0bfa011498c9824e15f9bb0a1cc, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/e6dc686fbfb648cea2ea4e7b4e687978, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/0df6b336cf9d49aeac68a8028620f12c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/6f96749bdd3c47bdafdd36fe2148b43c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/be35989a9d02441c8f7c04d7c1d9d9dc, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/c1d624e35cf64d2aad4d7f29e6408fad, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/b290e0f7389f4875883717721c12c6e9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/bf06c24f7b704e0d8374b58186c581e8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/2a590f2441bf41cc87d2c4659faac664, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/c6fefa83ffe04b0bba450c7ba1674e78, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/d94535be41e54a4aae59c626daab0fa8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/4d3db8f1b206489780cb5c530ef39879, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/0da37b7b23ba4a08ac533cd2da749f91, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/ba64f62674d8407590170facbfe01cc4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/1e2a1551df074894badb15bdb4154712, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/a3b080542cb440299b5867fa59e57c8c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/38749730addf4269bf3b0732da446f36, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/d199d1f2878b4e8d8307bb1280b09f17, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/830ee678e7654e08bcb1ee6e69cf76d5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/348cacb1bf2a4a65b6f668ebffe13cb4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/9fe10d04946540e7a88860b5caab746f] to archive 2024-11-13T05:40:44,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:40:44,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/74855d58134549389a45ac046294f2f6 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/74855d58134549389a45ac046294f2f6 2024-11-13T05:40:44,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/3e723dd5e41f42f3be5c0ffcd39fe5ea to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/3e723dd5e41f42f3be5c0ffcd39fe5ea 2024-11-13T05:40:44,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/8c500af0fe5f4e1db96427f5444a1f91 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/8c500af0fe5f4e1db96427f5444a1f91 2024-11-13T05:40:44,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/2715aa1684664b6b958aa4d021724a67 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/2715aa1684664b6b958aa4d021724a67 2024-11-13T05:40:44,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/3f395de68f284337b1db66ac44195a17 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/3f395de68f284337b1db66ac44195a17 2024-11-13T05:40:44,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/e186c0bfa011498c9824e15f9bb0a1cc to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/e186c0bfa011498c9824e15f9bb0a1cc 2024-11-13T05:40:44,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/e6dc686fbfb648cea2ea4e7b4e687978 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/e6dc686fbfb648cea2ea4e7b4e687978 2024-11-13T05:40:44,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/0df6b336cf9d49aeac68a8028620f12c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/0df6b336cf9d49aeac68a8028620f12c 2024-11-13T05:40:44,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/6f96749bdd3c47bdafdd36fe2148b43c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/6f96749bdd3c47bdafdd36fe2148b43c 2024-11-13T05:40:44,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/be35989a9d02441c8f7c04d7c1d9d9dc to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/be35989a9d02441c8f7c04d7c1d9d9dc 2024-11-13T05:40:44,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/c1d624e35cf64d2aad4d7f29e6408fad to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/c1d624e35cf64d2aad4d7f29e6408fad 2024-11-13T05:40:44,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/b290e0f7389f4875883717721c12c6e9 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/b290e0f7389f4875883717721c12c6e9 2024-11-13T05:40:44,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/bf06c24f7b704e0d8374b58186c581e8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/bf06c24f7b704e0d8374b58186c581e8 2024-11-13T05:40:44,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/2a590f2441bf41cc87d2c4659faac664 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/2a590f2441bf41cc87d2c4659faac664 2024-11-13T05:40:44,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/c6fefa83ffe04b0bba450c7ba1674e78 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/c6fefa83ffe04b0bba450c7ba1674e78 2024-11-13T05:40:44,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/d94535be41e54a4aae59c626daab0fa8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/d94535be41e54a4aae59c626daab0fa8 2024-11-13T05:40:44,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/4d3db8f1b206489780cb5c530ef39879 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/4d3db8f1b206489780cb5c530ef39879 2024-11-13T05:40:44,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/0da37b7b23ba4a08ac533cd2da749f91 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/0da37b7b23ba4a08ac533cd2da749f91 2024-11-13T05:40:44,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/ba64f62674d8407590170facbfe01cc4 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/ba64f62674d8407590170facbfe01cc4 2024-11-13T05:40:44,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/1e2a1551df074894badb15bdb4154712 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/1e2a1551df074894badb15bdb4154712 2024-11-13T05:40:44,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/a3b080542cb440299b5867fa59e57c8c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/a3b080542cb440299b5867fa59e57c8c 2024-11-13T05:40:44,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/38749730addf4269bf3b0732da446f36 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/38749730addf4269bf3b0732da446f36 2024-11-13T05:40:44,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/d199d1f2878b4e8d8307bb1280b09f17 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/d199d1f2878b4e8d8307bb1280b09f17 2024-11-13T05:40:44,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/830ee678e7654e08bcb1ee6e69cf76d5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/830ee678e7654e08bcb1ee6e69cf76d5 2024-11-13T05:40:44,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/348cacb1bf2a4a65b6f668ebffe13cb4 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/348cacb1bf2a4a65b6f668ebffe13cb4 2024-11-13T05:40:44,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/9fe10d04946540e7a88860b5caab746f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/9fe10d04946540e7a88860b5caab746f 2024-11-13T05:40:44,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/2843ad337531427ca8b6910fcf18dc9f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/84508c151e8346aba55e13349ee14e57, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/86df8e0def9b4a7f96e6fa6df6ecf42d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/684c8def7c40490a99a38ef27b7f1504, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d143e3e520aa4eaa9e22703b030ea99b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d8df68e7d22f4188a3e2d84d52fa47f9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/de69703fd4194f139009af8f39fefb31, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/4c356dede88f4550957df0285712bbbb, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d9a0d366460047b4bf4124127ce77167, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/92dd2875e8b74fcf9c3ba4df1f704f6b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/91d2894a093c4c8090a15d528b568758, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/b4fa2218225e433a8ee2bad8662a9408, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/408dcc5bb46b4075981dfc1797b35079, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/698293e53aa748bea999e98bdff51744, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/263b6fcfcd2e4391be9929c2f732a915, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/95f11b08a9f24c489d01fd16c533dd49, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/89faf4fbd7d04b2dadf0d1d51ca93a7f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/8694968fb5684b5ea34e6ae4653da063, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1db5e527cdbb49529f06bb2b3f708c85, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/c69e96e2937f48709a4158cd10b98151, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/ca0e6fe55b13458b87657353fb3bc2e3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1ec585b381034581979865d4e6fc18ae, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/f147c47e47cb47419f4f0ceaa676ebbe, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/59edf1c072494c9b9b6b5fd9b831b7c6, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/293fe46b34874e3c85626715c22b5e75, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d8ea5b0f311c40ab8f358e147d7d5a23] to archive 2024-11-13T05:40:44,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:40:44,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/2843ad337531427ca8b6910fcf18dc9f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/2843ad337531427ca8b6910fcf18dc9f 2024-11-13T05:40:44,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/84508c151e8346aba55e13349ee14e57 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/84508c151e8346aba55e13349ee14e57 2024-11-13T05:40:44,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/86df8e0def9b4a7f96e6fa6df6ecf42d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/86df8e0def9b4a7f96e6fa6df6ecf42d 2024-11-13T05:40:44,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/684c8def7c40490a99a38ef27b7f1504 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/684c8def7c40490a99a38ef27b7f1504 2024-11-13T05:40:44,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d143e3e520aa4eaa9e22703b030ea99b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d143e3e520aa4eaa9e22703b030ea99b 2024-11-13T05:40:44,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d8df68e7d22f4188a3e2d84d52fa47f9 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d8df68e7d22f4188a3e2d84d52fa47f9 2024-11-13T05:40:44,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/de69703fd4194f139009af8f39fefb31 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/de69703fd4194f139009af8f39fefb31 2024-11-13T05:40:44,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/4c356dede88f4550957df0285712bbbb to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/4c356dede88f4550957df0285712bbbb 2024-11-13T05:40:44,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d9a0d366460047b4bf4124127ce77167 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d9a0d366460047b4bf4124127ce77167 2024-11-13T05:40:44,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/92dd2875e8b74fcf9c3ba4df1f704f6b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/92dd2875e8b74fcf9c3ba4df1f704f6b 2024-11-13T05:40:44,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/91d2894a093c4c8090a15d528b568758 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/91d2894a093c4c8090a15d528b568758 2024-11-13T05:40:44,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/b4fa2218225e433a8ee2bad8662a9408 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/b4fa2218225e433a8ee2bad8662a9408 2024-11-13T05:40:44,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/408dcc5bb46b4075981dfc1797b35079 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/408dcc5bb46b4075981dfc1797b35079 2024-11-13T05:40:44,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/698293e53aa748bea999e98bdff51744 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/698293e53aa748bea999e98bdff51744 2024-11-13T05:40:44,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/263b6fcfcd2e4391be9929c2f732a915 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/263b6fcfcd2e4391be9929c2f732a915 2024-11-13T05:40:44,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/95f11b08a9f24c489d01fd16c533dd49 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/95f11b08a9f24c489d01fd16c533dd49 2024-11-13T05:40:44,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/89faf4fbd7d04b2dadf0d1d51ca93a7f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/89faf4fbd7d04b2dadf0d1d51ca93a7f 2024-11-13T05:40:44,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/8694968fb5684b5ea34e6ae4653da063 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/8694968fb5684b5ea34e6ae4653da063 2024-11-13T05:40:44,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1db5e527cdbb49529f06bb2b3f708c85 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1db5e527cdbb49529f06bb2b3f708c85 2024-11-13T05:40:44,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/c69e96e2937f48709a4158cd10b98151 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/c69e96e2937f48709a4158cd10b98151 2024-11-13T05:40:44,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/ca0e6fe55b13458b87657353fb3bc2e3 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/ca0e6fe55b13458b87657353fb3bc2e3 2024-11-13T05:40:44,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1ec585b381034581979865d4e6fc18ae to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1ec585b381034581979865d4e6fc18ae 2024-11-13T05:40:44,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/f147c47e47cb47419f4f0ceaa676ebbe to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/f147c47e47cb47419f4f0ceaa676ebbe 2024-11-13T05:40:44,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/59edf1c072494c9b9b6b5fd9b831b7c6 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/59edf1c072494c9b9b6b5fd9b831b7c6 2024-11-13T05:40:44,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/293fe46b34874e3c85626715c22b5e75 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/293fe46b34874e3c85626715c22b5e75 2024-11-13T05:40:44,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d8ea5b0f311c40ab8f358e147d7d5a23 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/d8ea5b0f311c40ab8f358e147d7d5a23 2024-11-13T05:40:44,326 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/recovered.edits/407.seqid, newMaxSeqId=407, maxSeqId=4 2024-11-13T05:40:44,326 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. 2024-11-13T05:40:44,327 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1676): Region close journal for 2aab469b292723a934ce524f70b46f92: Waiting for close lock at 1731476442563Running coprocessor pre-close hooks at 1731476442563Disabling compacts and flushes for region at 1731476442563Disabling writes for close at 1731476442563Obtaining lock to block concurrent updates at 1731476442563Preparing flush snapshotting stores in 2aab469b292723a934ce524f70b46f92 at 1731476442563Finished memstore snapshotting TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731476442563Flushing stores of TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92. at 1731476442564 (+1 ms)Flushing 2aab469b292723a934ce524f70b46f92/A: creating writer at 1731476442564Flushing mob file 2aab469b292723a934ce524f70b46f92/A: appending metadata at 1731476442570 (+6 ms)Flushing mob file 2aab469b292723a934ce524f70b46f92/A: closing flushed file at 1731476442570Flushing 2aab469b292723a934ce524f70b46f92/A: appending metadata at 1731476442983 (+413 ms)Flushing 2aab469b292723a934ce524f70b46f92/A: closing flushed file at 1731476442983Flushing 2aab469b292723a934ce524f70b46f92/B: creating writer at 1731476443400 (+417 ms)Flushing 2aab469b292723a934ce524f70b46f92/B: appending metadata at 1731476443405 (+5 ms)Flushing 2aab469b292723a934ce524f70b46f92/B: closing flushed file at 1731476443405Flushing 2aab469b292723a934ce524f70b46f92/C: creating writer at 1731476443819 (+414 ms)Flushing 2aab469b292723a934ce524f70b46f92/C: appending metadata at 1731476443823 (+4 ms)Flushing 2aab469b292723a934ce524f70b46f92/C: closing flushed file at 1731476443823Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e4ccc59: reopening flushed file at 1731476444239 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ce2f224: reopening flushed file at 1731476444243 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72e1e3d7: reopening flushed file at 1731476444248 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 2aab469b292723a934ce524f70b46f92 in 1689ms, sequenceid=404, compaction requested=true at 1731476444252 (+4 ms)Writing region close event to WAL at 1731476444323 (+71 ms)Running coprocessor post-close hooks at 1731476444326 (+3 ms)Closed at 1731476444326 2024-11-13T05:40:44,328 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(157): Closed 2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,329 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=48 updating hbase:meta row=2aab469b292723a934ce524f70b46f92, regionState=CLOSED 2024-11-13T05:40:44,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=48, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:40:44,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-13T05:40:44,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseRegionProcedure 2aab469b292723a934ce524f70b46f92, server=bc386b7bf2fe,45837,1731476380128 in 1.9200 sec 2024-11-13T05:40:44,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-11-13T05:40:44,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2aab469b292723a934ce524f70b46f92, UNASSIGN in 1.9240 sec 2024-11-13T05:40:44,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-13T05:40:44,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9270 sec 2024-11-13T05:40:44,335 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476444335"}]},"ts":"1731476444335"} 2024-11-13T05:40:44,336 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T05:40:44,336 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T05:40:44,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9360 sec 2024-11-13T05:40:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T05:40:44,547 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:44,548 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T05:40:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:44,550 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=50, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T05:40:44,551 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=50, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:44,553 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,556 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/recovered.edits] 2024-11-13T05:40:44,558 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/92d1f111f5ab48678228fb35e7a14e1f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/92d1f111f5ab48678228fb35e7a14e1f 2024-11-13T05:40:44,559 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/a8b4e0afc63048ac9c836965538dbee4 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/a8b4e0afc63048ac9c836965538dbee4 2024-11-13T05:40:44,560 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/e0db031d03f84eb49dddcf2b3889e254 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/A/e0db031d03f84eb49dddcf2b3889e254 2024-11-13T05:40:44,562 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/368b35ebfe0c4e7595b26c2bb4a17f75 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/368b35ebfe0c4e7595b26c2bb4a17f75 2024-11-13T05:40:44,564 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/645efa2cbfb24f9593cffd07545197bf to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/645efa2cbfb24f9593cffd07545197bf 2024-11-13T05:40:44,565 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/788c872ee2f0466690314857c84260ab to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/B/788c872ee2f0466690314857c84260ab 2024-11-13T05:40:44,567 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1868e1120e284858b941cbe9cea1a9ab to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/1868e1120e284858b941cbe9cea1a9ab 2024-11-13T05:40:44,569 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/38e8db17ade64a68a6c4c18d472dbad1 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/38e8db17ade64a68a6c4c18d472dbad1 2024-11-13T05:40:44,570 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/bd8c73353b204767bb0b1eb71af4a57d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/C/bd8c73353b204767bb0b1eb71af4a57d 2024-11-13T05:40:44,573 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/recovered.edits/407.seqid to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92/recovered.edits/407.seqid 2024-11-13T05:40:44,574 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,574 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T05:40:44,574 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T05:40:44,575 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T05:40:44,579 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130bb7c5c8b7054967bb1143b9bf812c91_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130bb7c5c8b7054967bb1143b9bf812c91_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,581 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130c4a4a56922a4b089f3659cc51779168_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130c4a4a56922a4b089f3659cc51779168_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,582 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111310df4568ec7a4539af0c9be6d604dafa_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111310df4568ec7a4539af0c9be6d604dafa_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,584 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111312a7653f47af405b9532da180ca7026e_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111312a7653f47af405b9532da180ca7026e_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,586 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113295553f501c74e6f9aa59a8ba9132d04_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113295553f501c74e6f9aa59a8ba9132d04_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,588 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111331ac143ce21e4173aacb6ed526c76bf5_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111331ac143ce21e4173aacb6ed526c76bf5_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,589 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111341bb967bd0e64f26952d6e1d7590a6d1_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111341bb967bd0e64f26952d6e1d7590a6d1_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,590 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134cf7c045334b4de89d46d3dfb0137305_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134cf7c045334b4de89d46d3dfb0137305_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,591 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135ada5b74577e408083844a18cc8a1884_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135ada5b74577e408083844a18cc8a1884_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,592 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136e5f4a853cf64647bfae1a1984346775_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136e5f4a853cf64647bfae1a1984346775_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,594 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138a4396f2fdaf457cb27e2eba3e0f6f1a_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138a4396f2fdaf457cb27e2eba3e0f6f1a_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,595 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138b926bc8d5e74d5492b94ef6f4a78197_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138b926bc8d5e74d5492b94ef6f4a78197_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,596 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111391c887dd36ee4007bb98233ab16b58c3_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111391c887dd36ee4007bb98233ab16b58c3_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,597 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139c817406ee1043ca8617997a17a89083_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139c817406ee1043ca8617997a17a89083_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,598 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139d5a272fd42d430094505dc46696baaa_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139d5a272fd42d430094505dc46696baaa_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,600 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a3a50d37872548e183ec6e7a7ae2f935_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a3a50d37872548e183ec6e7a7ae2f935_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,602 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c00f5a8347e844638997b13e0e9766ff_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c00f5a8347e844638997b13e0e9766ff_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,603 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c60c163e37b84f5ea2f31dc2da6635cc_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c60c163e37b84f5ea2f31dc2da6635cc_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,604 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d83c2e43cdd1403d9f062a49f576d7db_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d83c2e43cdd1403d9f062a49f576d7db_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,605 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e60529325eb044b89dae44f5cfafd99e_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e60529325eb044b89dae44f5cfafd99e_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,606 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ea9afc720f634a4abbef289e7a70378e_2aab469b292723a934ce524f70b46f92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ea9afc720f634a4abbef289e7a70378e_2aab469b292723a934ce524f70b46f92 2024-11-13T05:40:44,606 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T05:40:44,608 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=50, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:44,611 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T05:40:44,614 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T05:40:44,615 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=50, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:44,616 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T05:40:44,616 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731476444616"}]},"ts":"9223372036854775807"} 2024-11-13T05:40:44,618 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T05:40:44,618 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2aab469b292723a934ce524f70b46f92, NAME => 'TestAcidGuarantees,,1731476416741.2aab469b292723a934ce524f70b46f92.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T05:40:44,618 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T05:40:44,619 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731476444618"}]},"ts":"9223372036854775807"} 2024-11-13T05:40:44,621 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T05:40:44,621 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=50, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:44,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 73 msec 2024-11-13T05:40:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T05:40:44,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T05:40:44,656 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:44,666 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=240 (was 237) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1793353738_22 at /127.0.0.1:55320 [Waiting for operation #277] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/cluster_5e8179f6-de47-de9a-0430-c54b93078463/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-782295199_22 at /127.0.0.1:55244 [Waiting for operation #309] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/cluster_5e8179f6-de47-de9a-0430-c54b93078463/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=539 (was 504) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7266 (was 7603) 2024-11-13T05:40:44,675 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=539, ProcessCount=11, AvailableMemoryMB=7266 2024-11-13T05:40:44,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:40:44,676 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:40:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T05:40:44,678 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:40:44,678 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:44,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 51 2024-11-13T05:40:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-13T05:40:44,679 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:40:44,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742070_1246 (size=959) 2024-11-13T05:40:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-13T05:40:44,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-13T05:40:45,087 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745 2024-11-13T05:40:45,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742071_1247 (size=53) 2024-11-13T05:40:45,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-13T05:40:45,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:40:45,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f6767d6df0e08ef8c3ac8d792634f477, disabling compactions & flushes 2024-11-13T05:40:45,495 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:45,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:45,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. after waiting 0 ms 2024-11-13T05:40:45,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:45,495 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:45,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f6767d6df0e08ef8c3ac8d792634f477: Waiting for close lock at 1731476445495Disabling compacts and flushes for region at 1731476445495Disabling writes for close at 1731476445495Writing region close event to WAL at 1731476445495Closed at 1731476445495 2024-11-13T05:40:45,497 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:40:45,497 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731476445497"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731476445497"}]},"ts":"1731476445497"} 2024-11-13T05:40:45,499 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:40:45,500 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:40:45,500 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476445500"}]},"ts":"1731476445500"} 2024-11-13T05:40:45,502 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T05:40:45,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6767d6df0e08ef8c3ac8d792634f477, ASSIGN}] 2024-11-13T05:40:45,503 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6767d6df0e08ef8c3ac8d792634f477, ASSIGN 2024-11-13T05:40:45,504 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6767d6df0e08ef8c3ac8d792634f477, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45837,1731476380128; forceNewPlan=false, retain=false 2024-11-13T05:40:45,655 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=f6767d6df0e08ef8c3ac8d792634f477, regionState=OPENING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:45,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6767d6df0e08ef8c3ac8d792634f477, ASSIGN because future has completed 2024-11-13T05:40:45,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:40:45,813 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:45,813 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:40:45,813 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:45,813 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:40:45,813 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:45,813 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:45,815 INFO [StoreOpener-f6767d6df0e08ef8c3ac8d792634f477-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-13T05:40:45,830 INFO [StoreOpener-f6767d6df0e08ef8c3ac8d792634f477-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6767d6df0e08ef8c3ac8d792634f477 columnFamilyName A 2024-11-13T05:40:45,830 DEBUG [StoreOpener-f6767d6df0e08ef8c3ac8d792634f477-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:45,830 INFO [StoreOpener-f6767d6df0e08ef8c3ac8d792634f477-1 {}] regionserver.HStore(327): Store=f6767d6df0e08ef8c3ac8d792634f477/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:40:45,831 INFO [StoreOpener-f6767d6df0e08ef8c3ac8d792634f477-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:45,833 INFO [StoreOpener-f6767d6df0e08ef8c3ac8d792634f477-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6767d6df0e08ef8c3ac8d792634f477 columnFamilyName B 2024-11-13T05:40:45,833 DEBUG [StoreOpener-f6767d6df0e08ef8c3ac8d792634f477-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:45,834 INFO [StoreOpener-f6767d6df0e08ef8c3ac8d792634f477-1 {}] regionserver.HStore(327): Store=f6767d6df0e08ef8c3ac8d792634f477/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:40:45,834 INFO [StoreOpener-f6767d6df0e08ef8c3ac8d792634f477-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:45,836 INFO [StoreOpener-f6767d6df0e08ef8c3ac8d792634f477-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6767d6df0e08ef8c3ac8d792634f477 columnFamilyName C 2024-11-13T05:40:45,836 DEBUG [StoreOpener-f6767d6df0e08ef8c3ac8d792634f477-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:40:45,836 INFO [StoreOpener-f6767d6df0e08ef8c3ac8d792634f477-1 {}] regionserver.HStore(327): Store=f6767d6df0e08ef8c3ac8d792634f477/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:40:45,836 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:45,837 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:45,838 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:45,840 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:45,840 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:45,841 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:40:45,843 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:45,846 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:40:45,846 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened f6767d6df0e08ef8c3ac8d792634f477; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72848407, jitterRate=0.08552585542201996}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:40:45,847 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:45,847 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for f6767d6df0e08ef8c3ac8d792634f477: Running coprocessor pre-open hook at 1731476445813Writing region info on filesystem at 1731476445813Initializing all the Stores at 1731476445815 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476445815Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476445815Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476445815Cleaning up temporary data from old regions at 1731476445840 (+25 ms)Running coprocessor post-open hooks at 1731476445847 (+7 ms)Region opened successfully at 1731476445847 2024-11-13T05:40:45,848 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., pid=53, masterSystemTime=1731476445809 2024-11-13T05:40:45,851 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=f6767d6df0e08ef8c3ac8d792634f477, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:45,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:40:45,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-13T05:40:45,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; OpenRegionProcedure f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 in 198 msec 2024-11-13T05:40:45,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-13T05:40:45,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6767d6df0e08ef8c3ac8d792634f477, ASSIGN in 356 msec 2024-11-13T05:40:45,862 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:45,862 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:45,863 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:40:45,864 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476445863"}]},"ts":"1731476445863"} 2024-11-13T05:40:45,866 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T05:40:45,868 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:40:45,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1920 sec 2024-11-13T05:40:46,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-13T05:40:46,827 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:46,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c11f0a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:46,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:46,829 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:46,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38fd69da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:46,831 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c99ad4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,832 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:46,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:46,834 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:40:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,37913,1731476379290 2024-11-13T05:40:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3fb61d8d 2024-11-13T05:40:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:40:46,836 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:40:46,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a25e579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:46,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:46,838 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@226b4056, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:46,839 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:46,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3879309f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:46,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:46,841 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a65e02d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:46,843 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33dc76f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:46,844 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:46,845 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4c7a22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:46,846 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b68f1b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,847 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:46,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:46,848 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d3f8bdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:46,849 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d11b61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,850 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:46,850 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:46,851 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc7d13b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:46,853 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:46,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@405bcbd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,854 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:46,854 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:46,854 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d9cc4a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:46,855 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:46,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7df632, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:46,857 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:46,857 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fadac43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:46,858 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29596af2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:46,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:46,860 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5201384e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:46,861 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d8d4e3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,863 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:46,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:46,863 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70b4c9ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:46,864 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e96d664, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,865 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:40:46,866 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:40:46,866 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c32eb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:40:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:40:46,868 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:40:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fb9e60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:40:46,869 DEBUG [Thread-1165 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:46,869 DEBUG [Thread-1163 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:46,869 DEBUG [Thread-1152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:46,869 DEBUG [Thread-1150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:46,869 DEBUG [Thread-1146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:46,869 DEBUG [Thread-1157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:46,870 DEBUG [Thread-1148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:46,870 DEBUG [Thread-1161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:46,871 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:46,871 DEBUG [Thread-1154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:46,871 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:46,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:46,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:46,872 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:46,872 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:46,872 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:46,872 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:46,872 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:40:46,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-13T05:40:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T05:40:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:46,874 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:46,874 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:40:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:40:46,875 DEBUG [Thread-1159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:40:46,875 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:40:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:40:46,877 DEBUG [Thread-1159 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:40:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:40:46,877 DEBUG [Thread-1159 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:40:46,877 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:46,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:40:46,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:46,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:40:46,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:40:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:40:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:40:46,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:46,886 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:46,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:40:46,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/e8833732ec5e432a9bd839288cc2bb1c is 50, key is test_row_0/A:col10/1731476446884/Put/seqid=0 2024-11-13T05:40:46,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476506930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:46,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476506934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:46,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476506934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:46,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476506936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:46,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:46,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:46,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476506938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:46,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742072_1248 (size=12001) 2024-11-13T05:40:46,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/e8833732ec5e432a9bd839288cc2bb1c 2024-11-13T05:40:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T05:40:46,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/dae0ea5aa60b4e6482c9f1e1d8249c0e is 50, key is test_row_0/B:col10/1731476446884/Put/seqid=0 2024-11-13T05:40:46,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742073_1249 (size=12001) 2024-11-13T05:40:47,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T05:40:47,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:47,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:47,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:47,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476507046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476507046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476507047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476507047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476507047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T05:40:47,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T05:40:47,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:47,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:47,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476507256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476507256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476507257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476507257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476507257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T05:40:47,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:47,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:47,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:47,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/dae0ea5aa60b4e6482c9f1e1d8249c0e 2024-11-13T05:40:47,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/88ae212ee09641b7b2150ef1600bca3e is 50, key is test_row_0/C:col10/1731476446884/Put/seqid=0 2024-11-13T05:40:47,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742074_1250 (size=12001) 2024-11-13T05:40:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T05:40:47,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T05:40:47,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:47,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:47,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:47,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476507566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476507566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476507566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476507566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476507566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:47,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T05:40:47,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:47,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:47,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:47,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/88ae212ee09641b7b2150ef1600bca3e 2024-11-13T05:40:47,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T05:40:47,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/e8833732ec5e432a9bd839288cc2bb1c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/e8833732ec5e432a9bd839288cc2bb1c 2024-11-13T05:40:47,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:47,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:47,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:47,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:47,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/e8833732ec5e432a9bd839288cc2bb1c, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T05:40:47,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/dae0ea5aa60b4e6482c9f1e1d8249c0e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dae0ea5aa60b4e6482c9f1e1d8249c0e 2024-11-13T05:40:47,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dae0ea5aa60b4e6482c9f1e1d8249c0e, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T05:40:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/88ae212ee09641b7b2150ef1600bca3e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/88ae212ee09641b7b2150ef1600bca3e 2024-11-13T05:40:47,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/88ae212ee09641b7b2150ef1600bca3e, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T05:40:47,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f6767d6df0e08ef8c3ac8d792634f477 in 969ms, sequenceid=12, compaction requested=false 2024-11-13T05:40:47,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:47,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T05:40:47,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:47,995 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T05:40:48,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/a8dff56bc3174f20b8513132901e3f1c is 50, key is test_row_0/A:col10/1731476446930/Put/seqid=0 2024-11-13T05:40:48,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T05:40:48,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742075_1251 (size=12001) 2024-11-13T05:40:48,022 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/a8dff56bc3174f20b8513132901e3f1c 2024-11-13T05:40:48,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/dded9f13da9a4d07a19f704ae9fc9c60 is 50, key is test_row_0/B:col10/1731476446930/Put/seqid=0 2024-11-13T05:40:48,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742076_1252 (size=12001) 2024-11-13T05:40:48,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:48,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476508083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476508083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476508085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476508086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476508087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476508196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476508196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476508197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476508197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476508197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476508407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476508407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476508407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476508407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476508407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,453 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/dded9f13da9a4d07a19f704ae9fc9c60 2024-11-13T05:40:48,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/ee38c95bc2714faf9d5a31b570b98691 is 50, key is test_row_0/C:col10/1731476446930/Put/seqid=0 2024-11-13T05:40:48,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742077_1253 (size=12001) 2024-11-13T05:40:48,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476508716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476508716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476508716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476508716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476508716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:48,878 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/ee38c95bc2714faf9d5a31b570b98691 2024-11-13T05:40:48,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/a8dff56bc3174f20b8513132901e3f1c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/a8dff56bc3174f20b8513132901e3f1c 2024-11-13T05:40:48,887 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/a8dff56bc3174f20b8513132901e3f1c, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T05:40:48,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/dded9f13da9a4d07a19f704ae9fc9c60 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dded9f13da9a4d07a19f704ae9fc9c60 2024-11-13T05:40:48,894 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dded9f13da9a4d07a19f704ae9fc9c60, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T05:40:48,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/ee38c95bc2714faf9d5a31b570b98691 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/ee38c95bc2714faf9d5a31b570b98691 2024-11-13T05:40:48,900 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/ee38c95bc2714faf9d5a31b570b98691, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T05:40:48,901 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f6767d6df0e08ef8c3ac8d792634f477 in 906ms, sequenceid=37, compaction requested=false 2024-11-13T05:40:48,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:48,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:48,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-13T05:40:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-13T05:40:48,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-13T05:40:48,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0250 sec 2024-11-13T05:40:48,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.0330 sec 2024-11-13T05:40:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T05:40:49,017 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:49,017 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-13T05:40:49,019 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T05:40:49,019 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:49,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:49,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T05:40:49,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T05:40:49,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:49,172 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:40:49,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/609493020bf44bbb95f23737369830c0 is 50, key is test_row_0/A:col10/1731476448085/Put/seqid=0 2024-11-13T05:40:49,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742078_1254 (size=12001) 2024-11-13T05:40:49,179 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/609493020bf44bbb95f23737369830c0 2024-11-13T05:40:49,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/b8e7c78817344821aaf5330b9ecbcc22 is 50, key is test_row_0/B:col10/1731476448085/Put/seqid=0 2024-11-13T05:40:49,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742079_1255 (size=12001) 2024-11-13T05:40:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:49,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:49,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476509249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476509249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476509251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476509254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476509255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T05:40:49,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476509366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476509366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476509366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476509366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476509366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476509576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476509576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476509576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476509576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476509576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,613 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/b8e7c78817344821aaf5330b9ecbcc22 2024-11-13T05:40:49,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/80c2f278222b4a83a29b8844496d7f25 is 50, key is test_row_0/C:col10/1731476448085/Put/seqid=0 2024-11-13T05:40:49,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742080_1256 (size=12001) 2024-11-13T05:40:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T05:40:49,851 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:40:49,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476509887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476509887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476509887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476509887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:49,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476509887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:49,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:49,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:49,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,024 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/80c2f278222b4a83a29b8844496d7f25 2024-11-13T05:40:50,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/609493020bf44bbb95f23737369830c0 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/609493020bf44bbb95f23737369830c0 2024-11-13T05:40:50,033 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/609493020bf44bbb95f23737369830c0, entries=150, sequenceid=48, filesize=11.7 K 2024-11-13T05:40:50,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/b8e7c78817344821aaf5330b9ecbcc22 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/b8e7c78817344821aaf5330b9ecbcc22 2024-11-13T05:40:50,038 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/b8e7c78817344821aaf5330b9ecbcc22, entries=150, sequenceid=48, filesize=11.7 K 2024-11-13T05:40:50,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/80c2f278222b4a83a29b8844496d7f25 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/80c2f278222b4a83a29b8844496d7f25 2024-11-13T05:40:50,044 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/80c2f278222b4a83a29b8844496d7f25, entries=150, sequenceid=48, filesize=11.7 K 2024-11-13T05:40:50,045 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for f6767d6df0e08ef8c3ac8d792634f477 in 874ms, sequenceid=48, compaction requested=true 2024-11-13T05:40:50,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:50,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:50,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-13T05:40:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-13T05:40:50,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-13T05:40:50,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0280 sec 2024-11-13T05:40:50,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.0330 sec 2024-11-13T05:40:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T05:40:50,157 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:50,157 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-13T05:40:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T05:40:50,159 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:50,159 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:50,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:50,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T05:40:50,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T05:40:50,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:50,312 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T05:40:50,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/842305cb3bf742ad8ec71e063fea7036 is 50, key is test_row_0/A:col10/1731476449253/Put/seqid=0 2024-11-13T05:40:50,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742081_1257 (size=12001) 2024-11-13T05:40:50,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:50,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:50,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476510401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476510402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476510402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476510407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476510407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T05:40:50,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476510516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476510516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476510516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476510516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476510517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,722 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/842305cb3bf742ad8ec71e063fea7036 2024-11-13T05:40:50,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476510726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476510727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/dc591ac8c54344eda0edf8daa8a4d229 is 50, key is test_row_0/B:col10/1731476449253/Put/seqid=0 2024-11-13T05:40:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476510727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476510727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476510727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:50,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742082_1258 (size=12001) 2024-11-13T05:40:50,736 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/dc591ac8c54344eda0edf8daa8a4d229 2024-11-13T05:40:50,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/f2ac6e5e270048c0886ce45a325df0ca is 50, key is test_row_0/C:col10/1731476449253/Put/seqid=0 2024-11-13T05:40:50,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742083_1259 (size=12001) 2024-11-13T05:40:50,748 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/f2ac6e5e270048c0886ce45a325df0ca 2024-11-13T05:40:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/842305cb3bf742ad8ec71e063fea7036 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/842305cb3bf742ad8ec71e063fea7036 2024-11-13T05:40:50,766 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/842305cb3bf742ad8ec71e063fea7036, entries=150, sequenceid=74, filesize=11.7 K 2024-11-13T05:40:50,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/dc591ac8c54344eda0edf8daa8a4d229 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dc591ac8c54344eda0edf8daa8a4d229 2024-11-13T05:40:50,770 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dc591ac8c54344eda0edf8daa8a4d229, entries=150, sequenceid=74, filesize=11.7 K 2024-11-13T05:40:50,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/f2ac6e5e270048c0886ce45a325df0ca as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f2ac6e5e270048c0886ce45a325df0ca 2024-11-13T05:40:50,775 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f2ac6e5e270048c0886ce45a325df0ca, entries=150, sequenceid=74, filesize=11.7 K 2024-11-13T05:40:50,776 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f6767d6df0e08ef8c3ac8d792634f477 in 464ms, sequenceid=74, compaction requested=true 2024-11-13T05:40:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:50,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-13T05:40:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-13T05:40:50,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-13T05:40:50,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 619 msec 2024-11-13T05:40:50,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 623 msec 2024-11-13T05:40:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T05:40:50,786 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:50,787 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-13T05:40:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T05:40:50,788 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:50,789 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:50,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T05:40:50,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-13T05:40:50,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:50,941 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-13T05:40:50,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/640c7becab074e5ca740bec7db9c1e73 is 50, key is test_row_0/A:col10/1731476450405/Put/seqid=0 2024-11-13T05:40:50,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742084_1260 (size=12001) 2024-11-13T05:40:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:51,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:51,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476511065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476511071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476511097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476511097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476511097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T05:40:51,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476511206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476511206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476511206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476511206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476511207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,349 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/640c7becab074e5ca740bec7db9c1e73 2024-11-13T05:40:51,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/a7c1383efda442fc82aeba72132c8bad is 50, key is test_row_0/B:col10/1731476450405/Put/seqid=0 2024-11-13T05:40:51,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742085_1261 (size=12001) 2024-11-13T05:40:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T05:40:51,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476511417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476511417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476511418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476511418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476511418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476511726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476511727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476511727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476511727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476511736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:51,801 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/a7c1383efda442fc82aeba72132c8bad 2024-11-13T05:40:51,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/56db065403c6407c839069a85c6567e4 is 50, key is test_row_0/C:col10/1731476450405/Put/seqid=0 2024-11-13T05:40:51,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742086_1262 (size=12001) 2024-11-13T05:40:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T05:40:52,217 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/56db065403c6407c839069a85c6567e4 2024-11-13T05:40:52,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/640c7becab074e5ca740bec7db9c1e73 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/640c7becab074e5ca740bec7db9c1e73 2024-11-13T05:40:52,227 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/640c7becab074e5ca740bec7db9c1e73, entries=150, sequenceid=84, filesize=11.7 K 2024-11-13T05:40:52,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/a7c1383efda442fc82aeba72132c8bad as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/a7c1383efda442fc82aeba72132c8bad 2024-11-13T05:40:52,235 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/a7c1383efda442fc82aeba72132c8bad, entries=150, sequenceid=84, filesize=11.7 K 2024-11-13T05:40:52,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/56db065403c6407c839069a85c6567e4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/56db065403c6407c839069a85c6567e4 2024-11-13T05:40:52,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:52,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476512236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476512237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:52,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476512237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:52,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476512237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:52,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:52,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:52,244 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/56db065403c6407c839069a85c6567e4, entries=150, sequenceid=84, filesize=11.7 K 2024-11-13T05:40:52,245 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for f6767d6df0e08ef8c3ac8d792634f477 in 1304ms, sequenceid=84, compaction requested=true 2024-11-13T05:40:52,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:52,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:52,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-13T05:40:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-13T05:40:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:52,249 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-13T05:40:52,249 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4570 sec 2024-11-13T05:40:52,249 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:52,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-13T05:40:52,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.4640 sec 2024-11-13T05:40:52,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/c7710521427c47c39868cdef7a01851b is 50, key is test_row_0/A:col10/1731476452247/Put/seqid=0 2024-11-13T05:40:52,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742087_1263 (size=12001) 2024-11-13T05:40:52,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/c7710521427c47c39868cdef7a01851b 2024-11-13T05:40:52,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476512278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:52,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/d9b6f41219d64997bfbab5d340c3206a is 50, key is test_row_0/B:col10/1731476452247/Put/seqid=0 2024-11-13T05:40:52,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742088_1264 (size=12001) 2024-11-13T05:40:52,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/d9b6f41219d64997bfbab5d340c3206a 2024-11-13T05:40:52,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/99d40833d4214f4885b6f5f204168079 is 50, key is test_row_0/C:col10/1731476452247/Put/seqid=0 2024-11-13T05:40:52,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742089_1265 (size=12001) 2024-11-13T05:40:52,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/99d40833d4214f4885b6f5f204168079 2024-11-13T05:40:52,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/c7710521427c47c39868cdef7a01851b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/c7710521427c47c39868cdef7a01851b 2024-11-13T05:40:52,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/c7710521427c47c39868cdef7a01851b, entries=150, sequenceid=112, filesize=11.7 K 2024-11-13T05:40:52,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/d9b6f41219d64997bfbab5d340c3206a as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/d9b6f41219d64997bfbab5d340c3206a 2024-11-13T05:40:52,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476512386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:52,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/d9b6f41219d64997bfbab5d340c3206a, entries=150, sequenceid=112, filesize=11.7 K 2024-11-13T05:40:52,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/99d40833d4214f4885b6f5f204168079 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/99d40833d4214f4885b6f5f204168079 2024-11-13T05:40:52,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/99d40833d4214f4885b6f5f204168079, entries=150, sequenceid=112, filesize=11.7 K 2024-11-13T05:40:52,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for f6767d6df0e08ef8c3ac8d792634f477 in 153ms, sequenceid=112, compaction requested=true 2024-11-13T05:40:52,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:52,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:52,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:52,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:52,402 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-13T05:40:52,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:52,403 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-13T05:40:52,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:52,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:52,404 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-13T05:40:52,404 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-13T05:40:52,404 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/A is initiating minor compaction (all files) 2024-11-13T05:40:52,404 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/B is initiating minor compaction (all files) 2024-11-13T05:40:52,404 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/B in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:52,404 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/A in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:52,404 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dae0ea5aa60b4e6482c9f1e1d8249c0e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dded9f13da9a4d07a19f704ae9fc9c60, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/b8e7c78817344821aaf5330b9ecbcc22, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dc591ac8c54344eda0edf8daa8a4d229, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/a7c1383efda442fc82aeba72132c8bad, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/d9b6f41219d64997bfbab5d340c3206a] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=70.3 K 2024-11-13T05:40:52,404 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/e8833732ec5e432a9bd839288cc2bb1c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/a8dff56bc3174f20b8513132901e3f1c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/609493020bf44bbb95f23737369830c0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/842305cb3bf742ad8ec71e063fea7036, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/640c7becab074e5ca740bec7db9c1e73, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/c7710521427c47c39868cdef7a01851b] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=70.3 K 2024-11-13T05:40:52,405 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting dae0ea5aa60b4e6482c9f1e1d8249c0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731476446883 2024-11-13T05:40:52,405 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8833732ec5e432a9bd839288cc2bb1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731476446883 2024-11-13T05:40:52,405 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting dded9f13da9a4d07a19f704ae9fc9c60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731476446913 2024-11-13T05:40:52,405 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8dff56bc3174f20b8513132901e3f1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731476446913 2024-11-13T05:40:52,405 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting b8e7c78817344821aaf5330b9ecbcc22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731476448081 2024-11-13T05:40:52,406 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 609493020bf44bbb95f23737369830c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731476448081 2024-11-13T05:40:52,406 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting dc591ac8c54344eda0edf8daa8a4d229, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731476449246 2024-11-13T05:40:52,406 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting a7c1383efda442fc82aeba72132c8bad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731476450401 2024-11-13T05:40:52,406 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 842305cb3bf742ad8ec71e063fea7036, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731476449246 2024-11-13T05:40:52,406 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting d9b6f41219d64997bfbab5d340c3206a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731476451063 2024-11-13T05:40:52,406 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 640c7becab074e5ca740bec7db9c1e73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731476450401 2024-11-13T05:40:52,407 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7710521427c47c39868cdef7a01851b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731476451063 2024-11-13T05:40:52,419 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#A#compaction#219 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:52,420 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/85b119b248354f688b00e664fda6db6e is 50, key is test_row_0/A:col10/1731476452247/Put/seqid=0 2024-11-13T05:40:52,426 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#B#compaction#220 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:52,427 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/fee1d58419fc4f468eb22ac85d8bae92 is 50, key is test_row_0/B:col10/1731476452247/Put/seqid=0 2024-11-13T05:40:52,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742090_1266 (size=12207) 2024-11-13T05:40:52,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742091_1267 (size=12207) 2024-11-13T05:40:52,444 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/fee1d58419fc4f468eb22ac85d8bae92 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/fee1d58419fc4f468eb22ac85d8bae92 2024-11-13T05:40:52,451 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/B of f6767d6df0e08ef8c3ac8d792634f477 into fee1d58419fc4f468eb22ac85d8bae92(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:52,451 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:52,451 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/B, priority=10, startTime=1731476452402; duration=0sec 2024-11-13T05:40:52,451 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:52,451 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:B 2024-11-13T05:40:52,451 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-13T05:40:52,453 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-13T05:40:52,453 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/C is initiating minor compaction (all files) 2024-11-13T05:40:52,453 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/C in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:52,453 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/88ae212ee09641b7b2150ef1600bca3e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/ee38c95bc2714faf9d5a31b570b98691, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/80c2f278222b4a83a29b8844496d7f25, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f2ac6e5e270048c0886ce45a325df0ca, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/56db065403c6407c839069a85c6567e4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/99d40833d4214f4885b6f5f204168079] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=70.3 K 2024-11-13T05:40:52,454 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 88ae212ee09641b7b2150ef1600bca3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731476446883 2024-11-13T05:40:52,454 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting ee38c95bc2714faf9d5a31b570b98691, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731476446913 2024-11-13T05:40:52,455 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 80c2f278222b4a83a29b8844496d7f25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731476448081 2024-11-13T05:40:52,455 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting f2ac6e5e270048c0886ce45a325df0ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731476449246 2024-11-13T05:40:52,455 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 56db065403c6407c839069a85c6567e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731476450401 2024-11-13T05:40:52,456 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 99d40833d4214f4885b6f5f204168079, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731476451063 2024-11-13T05:40:52,467 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:52,468 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/32339951055c44b490afd88fbf6c487b is 50, key is test_row_0/C:col10/1731476452247/Put/seqid=0 2024-11-13T05:40:52,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742092_1268 (size=12207) 2024-11-13T05:40:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:52,612 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:52,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:40:52,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/5f8d0af2e41a439b91ef2a9f91cb2b93 is 50, key is test_row_0/A:col10/1731476452605/Put/seqid=0 2024-11-13T05:40:52,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742093_1269 (size=14341) 2024-11-13T05:40:52,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476512740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:52,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:52,838 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/85b119b248354f688b00e664fda6db6e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/85b119b248354f688b00e664fda6db6e 2024-11-13T05:40:52,843 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/A of f6767d6df0e08ef8c3ac8d792634f477 into 85b119b248354f688b00e664fda6db6e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:52,844 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:52,844 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/A, priority=10, startTime=1731476452402; duration=0sec 2024-11-13T05:40:52,844 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:52,844 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:A 2024-11-13T05:40:52,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476512857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:52,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:52,898 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/32339951055c44b490afd88fbf6c487b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/32339951055c44b490afd88fbf6c487b 2024-11-13T05:40:52,907 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/C of f6767d6df0e08ef8c3ac8d792634f477 into 32339951055c44b490afd88fbf6c487b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:52,907 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:52,907 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/C, priority=10, startTime=1731476452403; duration=0sec 2024-11-13T05:40:52,907 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:52,907 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:C 2024-11-13T05:40:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T05:40:52,936 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:52,936 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-13T05:40:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T05:40:52,940 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:52,941 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:52,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:53,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/5f8d0af2e41a439b91ef2a9f91cb2b93 2024-11-13T05:40:53,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T05:40:53,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/633cb3ff3a5f4757a6fdfb76b8db1280 is 50, key is test_row_0/B:col10/1731476452605/Put/seqid=0 2024-11-13T05:40:53,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742094_1270 (size=12001) 2024-11-13T05:40:53,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:53,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476513067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:53,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T05:40:53,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:53,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:53,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:53,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:53,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:53,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:53,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476513246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:53,248 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:53,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:53,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:53,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:53,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T05:40:53,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:53,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:53,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:53,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:53,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:53,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:53,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T05:40:53,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:53,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476513256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:53,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:53,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476513256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:53,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:53,258 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:53,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476513256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:53,259 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:53,259 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:53,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:53,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:53,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:53,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476513377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:53,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T05:40:53,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:53,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:53,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:53,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:53,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:53,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/633cb3ff3a5f4757a6fdfb76b8db1280 2024-11-13T05:40:53,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/309a63815d884bbab601fce03100f579 is 50, key is test_row_0/C:col10/1731476452605/Put/seqid=0 2024-11-13T05:40:53,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742095_1271 (size=12001) 2024-11-13T05:40:53,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/309a63815d884bbab601fce03100f579 2024-11-13T05:40:53,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/5f8d0af2e41a439b91ef2a9f91cb2b93 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/5f8d0af2e41a439b91ef2a9f91cb2b93 2024-11-13T05:40:53,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/5f8d0af2e41a439b91ef2a9f91cb2b93, entries=200, sequenceid=124, filesize=14.0 K 2024-11-13T05:40:53,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/633cb3ff3a5f4757a6fdfb76b8db1280 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/633cb3ff3a5f4757a6fdfb76b8db1280 2024-11-13T05:40:53,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/633cb3ff3a5f4757a6fdfb76b8db1280, entries=150, sequenceid=124, filesize=11.7 K 2024-11-13T05:40:53,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/309a63815d884bbab601fce03100f579 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/309a63815d884bbab601fce03100f579 2024-11-13T05:40:53,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/309a63815d884bbab601fce03100f579, entries=150, sequenceid=124, filesize=11.7 K 2024-11-13T05:40:53,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T05:40:53,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:53,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:53,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:53,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:53,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T05:40:53,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f6767d6df0e08ef8c3ac8d792634f477 in 954ms, sequenceid=124, compaction requested=false 2024-11-13T05:40:53,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:53,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T05:40:53,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:53,716 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T05:40:53,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/ea7d22302c734de38aee83cf90ee2252 is 50, key is test_row_0/A:col10/1731476452725/Put/seqid=0 2024-11-13T05:40:53,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742096_1272 (size=12151) 2024-11-13T05:40:53,759 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/ea7d22302c734de38aee83cf90ee2252 2024-11-13T05:40:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/33519e7408e04ea4913eb1dd0bc39536 is 50, key is test_row_0/B:col10/1731476452725/Put/seqid=0 2024-11-13T05:40:53,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742097_1273 (size=12151) 2024-11-13T05:40:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:53,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:53,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476513917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:53,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:53,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:53,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:54,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476514026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T05:40:54,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476514236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:54,242 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/33519e7408e04ea4913eb1dd0bc39536 2024-11-13T05:40:54,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/e56c686580ca4f4f9ea2be601e6fa213 is 50, key is test_row_0/C:col10/1731476452725/Put/seqid=0 2024-11-13T05:40:54,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742098_1274 (size=12151) 2024-11-13T05:40:54,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476514546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:54,662 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/e56c686580ca4f4f9ea2be601e6fa213 2024-11-13T05:40:54,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/ea7d22302c734de38aee83cf90ee2252 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/ea7d22302c734de38aee83cf90ee2252 2024-11-13T05:40:54,671 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/ea7d22302c734de38aee83cf90ee2252, entries=150, sequenceid=151, filesize=11.9 K 2024-11-13T05:40:54,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/33519e7408e04ea4913eb1dd0bc39536 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/33519e7408e04ea4913eb1dd0bc39536 2024-11-13T05:40:54,676 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/33519e7408e04ea4913eb1dd0bc39536, entries=150, sequenceid=151, filesize=11.9 K 2024-11-13T05:40:54,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/e56c686580ca4f4f9ea2be601e6fa213 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/e56c686580ca4f4f9ea2be601e6fa213 2024-11-13T05:40:54,681 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/e56c686580ca4f4f9ea2be601e6fa213, entries=150, sequenceid=151, filesize=11.9 K 2024-11-13T05:40:54,682 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f6767d6df0e08ef8c3ac8d792634f477 in 967ms, sequenceid=151, compaction requested=true 2024-11-13T05:40:54,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:54,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:54,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-13T05:40:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-13T05:40:54,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-13T05:40:54,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7430 sec 2024-11-13T05:40:54,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.7490 sec 2024-11-13T05:40:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:55,058 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:55,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T05:40:55,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/760ddb218f2e4d389a88ffdf02039b1c is 50, key is test_row_0/A:col10/1731476453916/Put/seqid=0 2024-11-13T05:40:55,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742099_1275 (size=12151) 2024-11-13T05:40:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T05:40:55,086 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:55,087 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-13T05:40:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T05:40:55,089 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:55,090 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:55,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:55,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476515148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T05:40:55,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T05:40:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:55,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476515257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:55,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476515266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:55,268 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:55,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476515276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:55,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476515276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:55,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:55,279 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476515276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:55,279 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:55,280 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:55,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T05:40:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:55,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T05:40:55,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476515466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:55,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/760ddb218f2e4d389a88ffdf02039b1c 2024-11-13T05:40:55,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/9aa7eb89e7d543e4943a2000c7622194 is 50, key is test_row_0/B:col10/1731476453916/Put/seqid=0 2024-11-13T05:40:55,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742100_1276 (size=12151) 2024-11-13T05:40:55,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T05:40:55,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:55,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:55,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:55,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T05:40:55,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:55,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T05:40:55,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476515776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:55,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:55,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T05:40:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:55,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:55,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/9aa7eb89e7d543e4943a2000c7622194 2024-11-13T05:40:55,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/962e6346578b421da21db21120ccb883 is 50, key is test_row_0/C:col10/1731476453916/Put/seqid=0 2024-11-13T05:40:55,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742101_1277 (size=12151) 2024-11-13T05:40:56,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T05:40:56,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:56,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:56,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:56,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:56,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:56,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T05:40:56,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:56,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:56,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T05:40:56,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476516287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:56,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:56,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:56,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:56,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T05:40:56,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:56,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:56,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:56,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:56,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:56,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/962e6346578b421da21db21120ccb883 2024-11-13T05:40:56,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/760ddb218f2e4d389a88ffdf02039b1c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/760ddb218f2e4d389a88ffdf02039b1c 2024-11-13T05:40:56,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/760ddb218f2e4d389a88ffdf02039b1c, entries=150, sequenceid=163, filesize=11.9 K 2024-11-13T05:40:56,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/9aa7eb89e7d543e4943a2000c7622194 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/9aa7eb89e7d543e4943a2000c7622194 2024-11-13T05:40:56,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/9aa7eb89e7d543e4943a2000c7622194, entries=150, sequenceid=163, filesize=11.9 K 2024-11-13T05:40:56,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/962e6346578b421da21db21120ccb883 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/962e6346578b421da21db21120ccb883 2024-11-13T05:40:56,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/962e6346578b421da21db21120ccb883, entries=150, sequenceid=163, filesize=11.9 K 2024-11-13T05:40:56,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f6767d6df0e08ef8c3ac8d792634f477 in 1278ms, sequenceid=163, compaction requested=true 2024-11-13T05:40:56,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:56,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:56,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:56,336 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:40:56,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:56,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:56,336 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:40:56,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:56,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:56,337 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:40:56,337 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:40:56,337 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/A is initiating minor compaction (all files) 2024-11-13T05:40:56,337 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/B is initiating minor compaction (all files) 2024-11-13T05:40:56,337 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/A in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:56,337 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/85b119b248354f688b00e664fda6db6e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/5f8d0af2e41a439b91ef2a9f91cb2b93, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/ea7d22302c734de38aee83cf90ee2252, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/760ddb218f2e4d389a88ffdf02039b1c] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=49.7 K 2024-11-13T05:40:56,337 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/B in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:56,337 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/fee1d58419fc4f468eb22ac85d8bae92, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/633cb3ff3a5f4757a6fdfb76b8db1280, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/33519e7408e04ea4913eb1dd0bc39536, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/9aa7eb89e7d543e4943a2000c7622194] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=47.4 K 2024-11-13T05:40:56,337 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85b119b248354f688b00e664fda6db6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731476451063 2024-11-13T05:40:56,338 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting fee1d58419fc4f468eb22ac85d8bae92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731476451063 2024-11-13T05:40:56,338 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f8d0af2e41a439b91ef2a9f91cb2b93, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731476452276 2024-11-13T05:40:56,338 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 633cb3ff3a5f4757a6fdfb76b8db1280, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731476452276 2024-11-13T05:40:56,338 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea7d22302c734de38aee83cf90ee2252, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731476452725 2024-11-13T05:40:56,338 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 760ddb218f2e4d389a88ffdf02039b1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731476453888 2024-11-13T05:40:56,338 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 33519e7408e04ea4913eb1dd0bc39536, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731476452725 2024-11-13T05:40:56,339 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aa7eb89e7d543e4943a2000c7622194, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731476453888 2024-11-13T05:40:56,345 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#A#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:56,345 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/05e0696c6f6a486cacc039c379cd3a6c is 50, key is test_row_0/A:col10/1731476453916/Put/seqid=0 2024-11-13T05:40:56,347 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:56,348 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/ca5fa28d5ed44fa39039dc4f3f944e84 is 50, key is test_row_0/B:col10/1731476453916/Put/seqid=0 2024-11-13T05:40:56,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742102_1278 (size=12493) 2024-11-13T05:40:56,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742103_1279 (size=12493) 2024-11-13T05:40:56,376 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/ca5fa28d5ed44fa39039dc4f3f944e84 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/ca5fa28d5ed44fa39039dc4f3f944e84 2024-11-13T05:40:56,380 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/B of f6767d6df0e08ef8c3ac8d792634f477 into ca5fa28d5ed44fa39039dc4f3f944e84(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:56,380 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:56,380 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/B, priority=12, startTime=1731476456336; duration=0sec 2024-11-13T05:40:56,380 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:56,380 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:B 2024-11-13T05:40:56,380 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:40:56,381 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:40:56,381 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/C is initiating minor compaction (all files) 2024-11-13T05:40:56,381 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/C in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:56,382 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/32339951055c44b490afd88fbf6c487b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/309a63815d884bbab601fce03100f579, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/e56c686580ca4f4f9ea2be601e6fa213, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/962e6346578b421da21db21120ccb883] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=47.4 K 2024-11-13T05:40:56,382 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 32339951055c44b490afd88fbf6c487b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731476451063 2024-11-13T05:40:56,382 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 309a63815d884bbab601fce03100f579, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731476452276 2024-11-13T05:40:56,382 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting e56c686580ca4f4f9ea2be601e6fa213, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731476452725 2024-11-13T05:40:56,382 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 962e6346578b421da21db21120ccb883, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731476453888 2024-11-13T05:40:56,389 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:56,389 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/a6fe23e6bc2648459a3d953e576b331f is 50, key is test_row_0/C:col10/1731476453916/Put/seqid=0 2024-11-13T05:40:56,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742104_1280 (size=12493) 2024-11-13T05:40:56,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T05:40:56,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:56,466 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T05:40:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/f05de721aefe4d87b42ce54c152c0b96 is 50, key is test_row_0/A:col10/1731476455127/Put/seqid=0 2024-11-13T05:40:56,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742105_1281 (size=12151) 2024-11-13T05:40:56,477 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/f05de721aefe4d87b42ce54c152c0b96 2024-11-13T05:40:56,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/9bded2abe76d4d1e9fe328e909dd7557 is 50, key is test_row_0/B:col10/1731476455127/Put/seqid=0 2024-11-13T05:40:56,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742106_1282 (size=12151) 2024-11-13T05:40:56,765 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/05e0696c6f6a486cacc039c379cd3a6c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/05e0696c6f6a486cacc039c379cd3a6c 2024-11-13T05:40:56,771 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/A of f6767d6df0e08ef8c3ac8d792634f477 into 05e0696c6f6a486cacc039c379cd3a6c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:56,771 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:56,771 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/A, priority=12, startTime=1731476456336; duration=0sec 2024-11-13T05:40:56,771 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:56,771 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:A 2024-11-13T05:40:56,814 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/a6fe23e6bc2648459a3d953e576b331f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/a6fe23e6bc2648459a3d953e576b331f 2024-11-13T05:40:56,823 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/C of f6767d6df0e08ef8c3ac8d792634f477 into a6fe23e6bc2648459a3d953e576b331f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:56,823 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:56,823 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/C, priority=12, startTime=1731476456336; duration=0sec 2024-11-13T05:40:56,823 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:56,824 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:C 2024-11-13T05:40:56,896 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/9bded2abe76d4d1e9fe328e909dd7557 2024-11-13T05:40:56,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/19af0724016e43329e3ef3a6beca54c9 is 50, key is test_row_0/C:col10/1731476455127/Put/seqid=0 2024-11-13T05:40:56,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742107_1283 (size=12151) 2024-11-13T05:40:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T05:40:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:57,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:57,340 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/19af0724016e43329e3ef3a6beca54c9 2024-11-13T05:40:57,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/f05de721aefe4d87b42ce54c152c0b96 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/f05de721aefe4d87b42ce54c152c0b96 2024-11-13T05:40:57,350 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/f05de721aefe4d87b42ce54c152c0b96, entries=150, sequenceid=188, filesize=11.9 K 2024-11-13T05:40:57,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/9bded2abe76d4d1e9fe328e909dd7557 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/9bded2abe76d4d1e9fe328e909dd7557 2024-11-13T05:40:57,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476517382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:57,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:57,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:57,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:57,392 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/9bded2abe76d4d1e9fe328e909dd7557, entries=150, sequenceid=188, filesize=11.9 K 2024-11-13T05:40:57,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/19af0724016e43329e3ef3a6beca54c9 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/19af0724016e43329e3ef3a6beca54c9 2024-11-13T05:40:57,398 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/19af0724016e43329e3ef3a6beca54c9, entries=150, sequenceid=188, filesize=11.9 K 2024-11-13T05:40:57,400 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f6767d6df0e08ef8c3ac8d792634f477 in 934ms, sequenceid=188, compaction requested=false 2024-11-13T05:40:57,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:57,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:57,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-13T05:40:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-13T05:40:57,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-13T05:40:57,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3110 sec 2024-11-13T05:40:57,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.3170 sec 2024-11-13T05:40:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:57,498 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:57,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T05:40:57,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/0dd3f28cff9a40909f9a630ab361f80d is 50, key is test_row_0/A:col10/1731476457497/Put/seqid=0 2024-11-13T05:40:57,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742108_1284 (size=12151) 2024-11-13T05:40:57,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/0dd3f28cff9a40909f9a630ab361f80d 2024-11-13T05:40:57,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/94102181fba74248a9a092b80daf9eb7 is 50, key is test_row_0/B:col10/1731476457497/Put/seqid=0 2024-11-13T05:40:57,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742109_1285 (size=12151) 2024-11-13T05:40:57,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/94102181fba74248a9a092b80daf9eb7 2024-11-13T05:40:57,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/6d5f4dd441e044b89012d8b0f45df374 is 50, key is test_row_0/C:col10/1731476457497/Put/seqid=0 2024-11-13T05:40:57,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742110_1286 (size=12151) 2024-11-13T05:40:57,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/6d5f4dd441e044b89012d8b0f45df374 2024-11-13T05:40:57,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476517633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:57,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:57,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/0dd3f28cff9a40909f9a630ab361f80d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/0dd3f28cff9a40909f9a630ab361f80d 2024-11-13T05:40:57,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/0dd3f28cff9a40909f9a630ab361f80d, entries=150, sequenceid=203, filesize=11.9 K 2024-11-13T05:40:57,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/94102181fba74248a9a092b80daf9eb7 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/94102181fba74248a9a092b80daf9eb7 2024-11-13T05:40:57,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/94102181fba74248a9a092b80daf9eb7, entries=150, sequenceid=203, filesize=11.9 K 2024-11-13T05:40:57,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/6d5f4dd441e044b89012d8b0f45df374 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/6d5f4dd441e044b89012d8b0f45df374 2024-11-13T05:40:57,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/6d5f4dd441e044b89012d8b0f45df374, entries=150, sequenceid=203, filesize=11.9 K 2024-11-13T05:40:57,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f6767d6df0e08ef8c3ac8d792634f477 in 161ms, sequenceid=203, compaction requested=true 2024-11-13T05:40:57,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:57,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:40:57,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:57,659 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:57,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:40:57,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:57,659 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:57,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:40:57,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:57,662 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:57,662 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/A is initiating minor compaction (all files) 2024-11-13T05:40:57,662 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:57,662 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/B is initiating minor compaction (all files) 2024-11-13T05:40:57,662 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/A in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:57,662 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/B in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:57,662 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/05e0696c6f6a486cacc039c379cd3a6c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/f05de721aefe4d87b42ce54c152c0b96, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/0dd3f28cff9a40909f9a630ab361f80d] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=35.9 K 2024-11-13T05:40:57,662 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/ca5fa28d5ed44fa39039dc4f3f944e84, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/9bded2abe76d4d1e9fe328e909dd7557, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/94102181fba74248a9a092b80daf9eb7] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=35.9 K 2024-11-13T05:40:57,663 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05e0696c6f6a486cacc039c379cd3a6c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731476453888 2024-11-13T05:40:57,664 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting ca5fa28d5ed44fa39039dc4f3f944e84, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731476453888 2024-11-13T05:40:57,664 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting f05de721aefe4d87b42ce54c152c0b96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731476455127 2024-11-13T05:40:57,664 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bded2abe76d4d1e9fe328e909dd7557, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731476455127 2024-11-13T05:40:57,665 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dd3f28cff9a40909f9a630ab361f80d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731476457327 2024-11-13T05:40:57,665 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 94102181fba74248a9a092b80daf9eb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731476457327 2024-11-13T05:40:57,682 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:57,682 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/5d65cd0ee3e24fda93c5f7d86838dc55 is 50, key is test_row_0/B:col10/1731476457497/Put/seqid=0 2024-11-13T05:40:57,693 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:57,693 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/d311864b246b406683a20f786d3a38e0 is 50, key is test_row_0/A:col10/1731476457497/Put/seqid=0 2024-11-13T05:40:57,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742111_1287 (size=12595) 2024-11-13T05:40:57,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742112_1288 (size=12595) 2024-11-13T05:40:57,719 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/d311864b246b406683a20f786d3a38e0 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/d311864b246b406683a20f786d3a38e0 2024-11-13T05:40:57,741 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/A of f6767d6df0e08ef8c3ac8d792634f477 into d311864b246b406683a20f786d3a38e0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:57,741 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:57,741 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/A, priority=13, startTime=1731476457659; duration=0sec 2024-11-13T05:40:57,741 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:40:57,741 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:A 2024-11-13T05:40:57,741 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:40:57,743 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:40:57,743 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/C is initiating minor compaction (all files) 2024-11-13T05:40:57,744 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/C in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:57,744 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/a6fe23e6bc2648459a3d953e576b331f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/19af0724016e43329e3ef3a6beca54c9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/6d5f4dd441e044b89012d8b0f45df374] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=35.9 K 2024-11-13T05:40:57,744 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6fe23e6bc2648459a3d953e576b331f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731476453888 2024-11-13T05:40:57,745 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19af0724016e43329e3ef3a6beca54c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731476455127 2024-11-13T05:40:57,745 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d5f4dd441e044b89012d8b0f45df374, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731476457327 2024-11-13T05:40:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:57,751 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:57,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T05:40:57,761 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#C#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:40:57,761 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/8785ffb489624f0582d93dc8ed350316 is 50, key is test_row_0/C:col10/1731476457497/Put/seqid=0 2024-11-13T05:40:57,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/75a084f10b0e4555a7c4d863c479c88e is 50, key is test_row_0/A:col10/1731476457603/Put/seqid=0 2024-11-13T05:40:57,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742113_1289 (size=12151) 2024-11-13T05:40:57,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742114_1290 (size=12595) 2024-11-13T05:40:57,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476517796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:57,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476517906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:58,112 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/5d65cd0ee3e24fda93c5f7d86838dc55 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/5d65cd0ee3e24fda93c5f7d86838dc55 2024-11-13T05:40:58,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476518124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:58,129 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/B of f6767d6df0e08ef8c3ac8d792634f477 into 5d65cd0ee3e24fda93c5f7d86838dc55(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:58,129 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:58,129 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/B, priority=13, startTime=1731476457659; duration=0sec 2024-11-13T05:40:58,130 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:58,130 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:B 2024-11-13T05:40:58,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/75a084f10b0e4555a7c4d863c479c88e 2024-11-13T05:40:58,202 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/8785ffb489624f0582d93dc8ed350316 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/8785ffb489624f0582d93dc8ed350316 2024-11-13T05:40:58,209 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/C of f6767d6df0e08ef8c3ac8d792634f477 into 8785ffb489624f0582d93dc8ed350316(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:40:58,209 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:58,210 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/C, priority=13, startTime=1731476457659; duration=0sec 2024-11-13T05:40:58,210 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:40:58,210 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:C 2024-11-13T05:40:58,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/81e2b61fa06a41f88c342c767254e9f5 is 50, key is test_row_0/B:col10/1731476457603/Put/seqid=0 2024-11-13T05:40:58,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742115_1291 (size=12151) 2024-11-13T05:40:58,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476518437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:58,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:58,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:58,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:58,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/81e2b61fa06a41f88c342c767254e9f5 2024-11-13T05:40:58,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/81d92576a3054369875fc56bd03da1ea is 50, key is test_row_0/C:col10/1731476457603/Put/seqid=0 2024-11-13T05:40:58,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742116_1292 (size=12151) 2024-11-13T05:40:58,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476518946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/81d92576a3054369875fc56bd03da1ea 2024-11-13T05:40:59,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/75a084f10b0e4555a7c4d863c479c88e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/75a084f10b0e4555a7c4d863c479c88e 2024-11-13T05:40:59,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/75a084f10b0e4555a7c4d863c479c88e, entries=150, sequenceid=228, filesize=11.9 K 2024-11-13T05:40:59,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/81e2b61fa06a41f88c342c767254e9f5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/81e2b61fa06a41f88c342c767254e9f5 2024-11-13T05:40:59,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/81e2b61fa06a41f88c342c767254e9f5, entries=150, sequenceid=228, filesize=11.9 K 2024-11-13T05:40:59,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/81d92576a3054369875fc56bd03da1ea as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/81d92576a3054369875fc56bd03da1ea 2024-11-13T05:40:59,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/81d92576a3054369875fc56bd03da1ea, entries=150, sequenceid=228, filesize=11.9 K 2024-11-13T05:40:59,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f6767d6df0e08ef8c3ac8d792634f477 in 1311ms, sequenceid=228, compaction requested=false 2024-11-13T05:40:59,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:40:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T05:40:59,257 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:40:59,257 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:40:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-13T05:40:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T05:40:59,259 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:40:59,260 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:40:59,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:40:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:40:59,287 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:40:59,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T05:40:59,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/56c40f8fd5c8442fbbbb12760b21cb04 is 50, key is test_row_0/A:col10/1731476457755/Put/seqid=0 2024-11-13T05:40:59,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742117_1293 (size=14541) 2024-11-13T05:40:59,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476519310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476519311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476519312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476519315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T05:40:59,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T05:40:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:59,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:59,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476519427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476519427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476519427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476519427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T05:40:59,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:59,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:59,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:59,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:59,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T05:40:59,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476519637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476519637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476519637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476519637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/56c40f8fd5c8442fbbbb12760b21cb04 2024-11-13T05:40:59,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/389713e33b3b40afa43b120d8d6166e9 is 50, key is test_row_0/B:col10/1731476457755/Put/seqid=0 2024-11-13T05:40:59,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742118_1294 (size=12151) 2024-11-13T05:40:59,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T05:40:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:59,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:59,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T05:40:59,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:59,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:40:59,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:40:59,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:59,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:40:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T05:40:59,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476519946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476519946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476519947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476519947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:40:59,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:40:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476519966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:40:59,969 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:40:59,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:40:59,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:00,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T05:41:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:00,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:00,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/389713e33b3b40afa43b120d8d6166e9 2024-11-13T05:41:00,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/fa16ad848d9d43359e0e5e2aad151951 is 50, key is test_row_0/C:col10/1731476457755/Put/seqid=0 2024-11-13T05:41:00,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742119_1295 (size=12151) 2024-11-13T05:41:00,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T05:41:00,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:00,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:00,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:00,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:00,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:00,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T05:41:00,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:00,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:00,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:00,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:00,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T05:41:00,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476520456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:00,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:00,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476520456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:00,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:00,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:00,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:00,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:00,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:00,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476520456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:00,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:00,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:00,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:00,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476520456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:00,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:00,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:00,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:00,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T05:41:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:00,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:00,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:00,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:00,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:00,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/fa16ad848d9d43359e0e5e2aad151951 2024-11-13T05:41:00,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/56c40f8fd5c8442fbbbb12760b21cb04 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/56c40f8fd5c8442fbbbb12760b21cb04 2024-11-13T05:41:00,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/56c40f8fd5c8442fbbbb12760b21cb04, entries=200, sequenceid=244, filesize=14.2 K 2024-11-13T05:41:00,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/389713e33b3b40afa43b120d8d6166e9 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/389713e33b3b40afa43b120d8d6166e9 2024-11-13T05:41:00,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/389713e33b3b40afa43b120d8d6166e9, entries=150, sequenceid=244, filesize=11.9 K 2024-11-13T05:41:00,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/fa16ad848d9d43359e0e5e2aad151951 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/fa16ad848d9d43359e0e5e2aad151951 2024-11-13T05:41:00,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/fa16ad848d9d43359e0e5e2aad151951, entries=150, sequenceid=244, filesize=11.9 K 2024-11-13T05:41:00,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f6767d6df0e08ef8c3ac8d792634f477 in 1246ms, sequenceid=244, compaction requested=true 2024-11-13T05:41:00,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:00,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:00,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:00,534 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:00,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:00,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:00,534 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:00,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:00,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:00,536 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:00,536 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:00,537 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/A is initiating minor compaction (all files) 2024-11-13T05:41:00,537 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/B is initiating minor compaction (all files) 2024-11-13T05:41:00,537 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/B in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:00,537 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/A in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:00,537 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/d311864b246b406683a20f786d3a38e0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/75a084f10b0e4555a7c4d863c479c88e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/56c40f8fd5c8442fbbbb12760b21cb04] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=38.4 K 2024-11-13T05:41:00,537 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/5d65cd0ee3e24fda93c5f7d86838dc55, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/81e2b61fa06a41f88c342c767254e9f5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/389713e33b3b40afa43b120d8d6166e9] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=36.0 K 2024-11-13T05:41:00,537 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting d311864b246b406683a20f786d3a38e0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731476457327 2024-11-13T05:41:00,537 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d65cd0ee3e24fda93c5f7d86838dc55, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731476457327 2024-11-13T05:41:00,537 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75a084f10b0e4555a7c4d863c479c88e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731476457603 2024-11-13T05:41:00,537 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e2b61fa06a41f88c342c767254e9f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731476457603 2024-11-13T05:41:00,537 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56c40f8fd5c8442fbbbb12760b21cb04, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731476457755 2024-11-13T05:41:00,537 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 389713e33b3b40afa43b120d8d6166e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731476457755 2024-11-13T05:41:00,543 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:00,543 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/f419204b8c1044298b72d4055a598597 is 50, key is test_row_0/B:col10/1731476457755/Put/seqid=0 2024-11-13T05:41:00,543 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:00,544 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/02dcba5914864d7095e438f747b79fe4 is 50, key is test_row_0/A:col10/1731476457755/Put/seqid=0 2024-11-13T05:41:00,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742120_1296 (size=12697) 2024-11-13T05:41:00,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742121_1297 (size=12697) 2024-11-13T05:41:00,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T05:41:00,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:00,634 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T05:41:00,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/4767e63eaf4e4a5b9c9034e965339629 is 50, key is test_row_0/A:col10/1731476459314/Put/seqid=0 2024-11-13T05:41:00,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742122_1298 (size=12301) 2024-11-13T05:41:00,952 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/f419204b8c1044298b72d4055a598597 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/f419204b8c1044298b72d4055a598597 2024-11-13T05:41:00,956 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/B of f6767d6df0e08ef8c3ac8d792634f477 into f419204b8c1044298b72d4055a598597(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:00,956 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:00,956 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/B, priority=13, startTime=1731476460534; duration=0sec 2024-11-13T05:41:00,956 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:00,956 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:B 2024-11-13T05:41:00,957 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:00,957 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:00,957 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/C is initiating minor compaction (all files) 2024-11-13T05:41:00,957 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/C in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:00,958 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/8785ffb489624f0582d93dc8ed350316, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/81d92576a3054369875fc56bd03da1ea, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/fa16ad848d9d43359e0e5e2aad151951] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=36.0 K 2024-11-13T05:41:00,958 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 8785ffb489624f0582d93dc8ed350316, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731476457327 2024-11-13T05:41:00,958 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 81d92576a3054369875fc56bd03da1ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731476457603 2024-11-13T05:41:00,958 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting fa16ad848d9d43359e0e5e2aad151951, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731476457755 2024-11-13T05:41:00,960 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/02dcba5914864d7095e438f747b79fe4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/02dcba5914864d7095e438f747b79fe4 2024-11-13T05:41:00,965 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/A of f6767d6df0e08ef8c3ac8d792634f477 into 02dcba5914864d7095e438f747b79fe4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:00,965 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:00,965 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/A, priority=13, startTime=1731476460534; duration=0sec 2024-11-13T05:41:00,965 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:00,965 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:A 2024-11-13T05:41:00,966 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:00,966 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/30d8bdc94b7d49cc85e04ff1e96d4b42 is 50, key is test_row_0/C:col10/1731476457755/Put/seqid=0 2024-11-13T05:41:00,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742123_1299 (size=12697) 2024-11-13T05:41:01,043 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/4767e63eaf4e4a5b9c9034e965339629 2024-11-13T05:41:01,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/bc06725c64274e15a83640133bd4eb2f is 50, key is test_row_0/B:col10/1731476459314/Put/seqid=0 2024-11-13T05:41:01,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742124_1300 (size=12301) 2024-11-13T05:41:01,374 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/30d8bdc94b7d49cc85e04ff1e96d4b42 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/30d8bdc94b7d49cc85e04ff1e96d4b42 2024-11-13T05:41:01,379 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/C of f6767d6df0e08ef8c3ac8d792634f477 into 30d8bdc94b7d49cc85e04ff1e96d4b42(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:01,379 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:01,379 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/C, priority=13, startTime=1731476460534; duration=0sec 2024-11-13T05:41:01,379 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:01,379 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:C 2024-11-13T05:41:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T05:41:01,453 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/bc06725c64274e15a83640133bd4eb2f 2024-11-13T05:41:01,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/851d15b4efb844d58c7b8c5fa172e4e2 is 50, key is test_row_0/C:col10/1731476459314/Put/seqid=0 2024-11-13T05:41:01,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742125_1301 (size=12301) 2024-11-13T05:41:01,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:41:01,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476521481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:01,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476521481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:01,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476521484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:01,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476521485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:01,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:01,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:01,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476521596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:01,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476521596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:01,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476521596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476521596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:01,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476521806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476521806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476521806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:01,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476521806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:01,864 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/851d15b4efb844d58c7b8c5fa172e4e2 2024-11-13T05:41:01,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/4767e63eaf4e4a5b9c9034e965339629 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/4767e63eaf4e4a5b9c9034e965339629 2024-11-13T05:41:01,872 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/4767e63eaf4e4a5b9c9034e965339629, entries=150, sequenceid=266, filesize=12.0 K 2024-11-13T05:41:01,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/bc06725c64274e15a83640133bd4eb2f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/bc06725c64274e15a83640133bd4eb2f 2024-11-13T05:41:01,877 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/bc06725c64274e15a83640133bd4eb2f, entries=150, sequenceid=266, filesize=12.0 K 2024-11-13T05:41:01,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/851d15b4efb844d58c7b8c5fa172e4e2 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/851d15b4efb844d58c7b8c5fa172e4e2 2024-11-13T05:41:01,881 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/851d15b4efb844d58c7b8c5fa172e4e2, entries=150, sequenceid=266, filesize=12.0 K 2024-11-13T05:41:01,882 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f6767d6df0e08ef8c3ac8d792634f477 in 1248ms, sequenceid=266, compaction requested=false 2024-11-13T05:41:01,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:01,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:01,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-13T05:41:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-13T05:41:01,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-13T05:41:01,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6230 sec 2024-11-13T05:41:01,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.6280 sec 2024-11-13T05:41:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:41:01,978 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:01,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-13T05:41:01,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/587612f52ea94de5a455c7c027afa279 is 50, key is test_row_0/A:col10/1731476461480/Put/seqid=0 2024-11-13T05:41:01,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742126_1302 (size=14741) 2024-11-13T05:41:02,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476522060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:02,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476522116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:02,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:02,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476522116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:02,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476522117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:02,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476522117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:02,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476522166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:02,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476522376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:02,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/587612f52ea94de5a455c7c027afa279 2024-11-13T05:41:02,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/4857bec2bde34b2287a859c11ca27b17 is 50, key is test_row_0/B:col10/1731476461480/Put/seqid=0 2024-11-13T05:41:02,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742127_1303 (size=12301) 2024-11-13T05:41:02,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/4857bec2bde34b2287a859c11ca27b17 2024-11-13T05:41:02,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/611f2f46bf164177b29ec39966a446c9 is 50, key is test_row_0/C:col10/1731476461480/Put/seqid=0 2024-11-13T05:41:02,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742128_1304 (size=12301) 2024-11-13T05:41:02,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476522627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:02,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:02,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476522627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:02,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476522627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476522627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:02,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476522687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:02,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/611f2f46bf164177b29ec39966a446c9 2024-11-13T05:41:02,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/587612f52ea94de5a455c7c027afa279 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/587612f52ea94de5a455c7c027afa279 2024-11-13T05:41:02,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/587612f52ea94de5a455c7c027afa279, entries=200, sequenceid=284, filesize=14.4 K 2024-11-13T05:41:02,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/4857bec2bde34b2287a859c11ca27b17 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4857bec2bde34b2287a859c11ca27b17 2024-11-13T05:41:02,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4857bec2bde34b2287a859c11ca27b17, entries=150, sequenceid=284, filesize=12.0 K 2024-11-13T05:41:02,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/611f2f46bf164177b29ec39966a446c9 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/611f2f46bf164177b29ec39966a446c9 2024-11-13T05:41:02,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/611f2f46bf164177b29ec39966a446c9, entries=150, sequenceid=284, filesize=12.0 K 2024-11-13T05:41:02,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for f6767d6df0e08ef8c3ac8d792634f477 in 884ms, sequenceid=284, compaction requested=true 2024-11-13T05:41:02,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:02,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:02,863 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:02,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:02,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:02,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:02,863 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:02,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:02,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:02,864 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:02,864 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:02,864 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/B is initiating minor compaction (all files) 2024-11-13T05:41:02,864 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/A is initiating minor compaction (all files) 2024-11-13T05:41:02,864 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/B in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:02,864 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/A in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:02,864 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/f419204b8c1044298b72d4055a598597, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/bc06725c64274e15a83640133bd4eb2f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4857bec2bde34b2287a859c11ca27b17] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=36.4 K 2024-11-13T05:41:02,864 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/02dcba5914864d7095e438f747b79fe4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/4767e63eaf4e4a5b9c9034e965339629, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/587612f52ea94de5a455c7c027afa279] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=38.8 K 2024-11-13T05:41:02,864 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting f419204b8c1044298b72d4055a598597, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731476457755 2024-11-13T05:41:02,864 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02dcba5914864d7095e438f747b79fe4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731476457755 2024-11-13T05:41:02,864 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting bc06725c64274e15a83640133bd4eb2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731476459310 2024-11-13T05:41:02,864 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4767e63eaf4e4a5b9c9034e965339629, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731476459310 2024-11-13T05:41:02,865 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 4857bec2bde34b2287a859c11ca27b17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731476461480 2024-11-13T05:41:02,865 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 587612f52ea94de5a455c7c027afa279, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731476461480 2024-11-13T05:41:02,871 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:02,871 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/46496f6983af46578e600abe9c3fbda4 is 50, key is test_row_0/B:col10/1731476461480/Put/seqid=0 2024-11-13T05:41:02,877 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#A#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:02,877 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/9e8e22ccefdb4351b2d2f1b1fae8499e is 50, key is test_row_0/A:col10/1731476461480/Put/seqid=0 2024-11-13T05:41:02,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742129_1305 (size=12949) 2024-11-13T05:41:02,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742130_1306 (size=12949) 2024-11-13T05:41:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:41:03,199 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:03,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T05:41:03,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/fe3d0d37041d47ea97a5e60c30cda08b is 50, key is test_row_0/A:col10/1731476462020/Put/seqid=0 2024-11-13T05:41:03,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742131_1307 (size=12301) 2024-11-13T05:41:03,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/fe3d0d37041d47ea97a5e60c30cda08b 2024-11-13T05:41:03,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/046bfdb5466b472f9b99b755d0bd48a0 is 50, key is test_row_0/B:col10/1731476462020/Put/seqid=0 2024-11-13T05:41:03,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742132_1308 (size=12301) 2024-11-13T05:41:03,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/046bfdb5466b472f9b99b755d0bd48a0 2024-11-13T05:41:03,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/4b722b8bac2d4abc8909cdf3eb821e24 is 50, key is test_row_0/C:col10/1731476462020/Put/seqid=0 2024-11-13T05:41:03,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742133_1309 (size=12301) 2024-11-13T05:41:03,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476523249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:03,288 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/46496f6983af46578e600abe9c3fbda4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/46496f6983af46578e600abe9c3fbda4 2024-11-13T05:41:03,292 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/9e8e22ccefdb4351b2d2f1b1fae8499e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/9e8e22ccefdb4351b2d2f1b1fae8499e 2024-11-13T05:41:03,292 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/B of f6767d6df0e08ef8c3ac8d792634f477 into 46496f6983af46578e600abe9c3fbda4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:03,292 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:03,292 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/B, priority=13, startTime=1731476462863; duration=0sec 2024-11-13T05:41:03,292 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:03,292 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:B 2024-11-13T05:41:03,292 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:03,294 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:03,294 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/C is initiating minor compaction (all files) 2024-11-13T05:41:03,294 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/C in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:03,294 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/30d8bdc94b7d49cc85e04ff1e96d4b42, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/851d15b4efb844d58c7b8c5fa172e4e2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/611f2f46bf164177b29ec39966a446c9] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=36.4 K 2024-11-13T05:41:03,294 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 30d8bdc94b7d49cc85e04ff1e96d4b42, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731476457755 2024-11-13T05:41:03,295 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 851d15b4efb844d58c7b8c5fa172e4e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731476459310 2024-11-13T05:41:03,295 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 611f2f46bf164177b29ec39966a446c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731476461480 2024-11-13T05:41:03,297 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/A of f6767d6df0e08ef8c3ac8d792634f477 into 9e8e22ccefdb4351b2d2f1b1fae8499e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:03,297 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:03,297 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/A, priority=13, startTime=1731476462863; duration=0sec 2024-11-13T05:41:03,297 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:03,297 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:A 2024-11-13T05:41:03,302 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#C#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:03,302 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/9cc178f1743f4bc8b4819ec6d9c70609 is 50, key is test_row_0/C:col10/1731476461480/Put/seqid=0 2024-11-13T05:41:03,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742134_1310 (size=12949) 2024-11-13T05:41:03,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476523356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T05:41:03,421 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:03,422 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-13T05:41:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T05:41:03,424 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:03,424 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:03,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T05:41:03,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476523566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:03,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T05:41:03,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:03,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:03,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:03,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:03,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:03,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/4b722b8bac2d4abc8909cdf3eb821e24 2024-11-13T05:41:03,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/fe3d0d37041d47ea97a5e60c30cda08b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/fe3d0d37041d47ea97a5e60c30cda08b 2024-11-13T05:41:03,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/fe3d0d37041d47ea97a5e60c30cda08b, entries=150, sequenceid=306, filesize=12.0 K 2024-11-13T05:41:03,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/046bfdb5466b472f9b99b755d0bd48a0 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/046bfdb5466b472f9b99b755d0bd48a0 2024-11-13T05:41:03,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476523637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:03,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476523637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:03,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:03,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/046bfdb5466b472f9b99b755d0bd48a0, entries=150, sequenceid=306, filesize=12.0 K 2024-11-13T05:41:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476523637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:03,639 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:03,640 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:03,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/4b722b8bac2d4abc8909cdf3eb821e24 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/4b722b8bac2d4abc8909cdf3eb821e24 2024-11-13T05:41:03,641 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:03,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/4b722b8bac2d4abc8909cdf3eb821e24, entries=150, sequenceid=306, filesize=12.0 K 2024-11-13T05:41:03,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f6767d6df0e08ef8c3ac8d792634f477 in 447ms, sequenceid=306, compaction requested=false 2024-11-13T05:41:03,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:41:03,649 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:03,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-13T05:41:03,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/c7338a32746c4332b496802f4bbdf6ca is 50, key is test_row_0/A:col10/1731476463242/Put/seqid=0 2024-11-13T05:41:03,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742135_1311 (size=14741) 2024-11-13T05:41:03,717 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/9cc178f1743f4bc8b4819ec6d9c70609 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/9cc178f1743f4bc8b4819ec6d9c70609 2024-11-13T05:41:03,725 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/C of f6767d6df0e08ef8c3ac8d792634f477 into 9cc178f1743f4bc8b4819ec6d9c70609(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:03,725 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:03,726 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/C, priority=13, startTime=1731476462863; duration=0sec 2024-11-13T05:41:03,726 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:03,726 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:C 2024-11-13T05:41:03,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T05:41:03,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:03,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:03,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:03,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:03,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T05:41:03,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476523739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:03,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:03,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476523846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:03,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:03,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476523876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:03,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T05:41:03,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:03,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:03,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:03,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:03,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T05:41:04,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:04,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T05:41:04,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476524056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:04,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/c7338a32746c4332b496802f4bbdf6ca 2024-11-13T05:41:04,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:04,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:04,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:04,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/4f12c178a6fa4ddfaa1a1df26883a809 is 50, key is test_row_0/B:col10/1731476463242/Put/seqid=0 2024-11-13T05:41:04,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742136_1312 (size=12301) 2024-11-13T05:41:04,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T05:41:04,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:04,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T05:41:04,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:04,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:04,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476524366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:04,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:04,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:04,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:04,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476524387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:04,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/4f12c178a6fa4ddfaa1a1df26883a809 2024-11-13T05:41:04,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/81fe53eba0e441ab943a41d64f34e181 is 50, key is test_row_0/C:col10/1731476463242/Put/seqid=0 2024-11-13T05:41:04,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742137_1313 (size=12301) 2024-11-13T05:41:04,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T05:41:04,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:04,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T05:41:04,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T05:41:04,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:04,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T05:41:04,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:04,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:04,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:04,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476524876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:04,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:04,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:04,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/81fe53eba0e441ab943a41d64f34e181 2024-11-13T05:41:04,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/c7338a32746c4332b496802f4bbdf6ca as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/c7338a32746c4332b496802f4bbdf6ca 2024-11-13T05:41:04,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/c7338a32746c4332b496802f4bbdf6ca, entries=200, sequenceid=323, filesize=14.4 K 2024-11-13T05:41:04,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/4f12c178a6fa4ddfaa1a1df26883a809 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4f12c178a6fa4ddfaa1a1df26883a809 2024-11-13T05:41:04,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4f12c178a6fa4ddfaa1a1df26883a809, entries=150, sequenceid=323, filesize=12.0 K 2024-11-13T05:41:04,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/81fe53eba0e441ab943a41d64f34e181 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/81fe53eba0e441ab943a41d64f34e181 2024-11-13T05:41:04,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/81fe53eba0e441ab943a41d64f34e181, entries=150, sequenceid=323, filesize=12.0 K 2024-11-13T05:41:04,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for f6767d6df0e08ef8c3ac8d792634f477 in 1255ms, sequenceid=323, compaction requested=true 2024-11-13T05:41:04,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:04,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:04,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:04,905 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:04,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:04,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:04,905 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:04,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:04,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:04,905 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:04,905 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:04,906 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/A is initiating minor compaction (all files) 2024-11-13T05:41:04,906 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/B is initiating minor compaction (all files) 2024-11-13T05:41:04,906 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/A in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,906 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/B in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,906 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/46496f6983af46578e600abe9c3fbda4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/046bfdb5466b472f9b99b755d0bd48a0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4f12c178a6fa4ddfaa1a1df26883a809] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=36.7 K 2024-11-13T05:41:04,906 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/9e8e22ccefdb4351b2d2f1b1fae8499e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/fe3d0d37041d47ea97a5e60c30cda08b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/c7338a32746c4332b496802f4bbdf6ca] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=39.1 K 2024-11-13T05:41:04,906 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e8e22ccefdb4351b2d2f1b1fae8499e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731476461480 2024-11-13T05:41:04,906 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 46496f6983af46578e600abe9c3fbda4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731476461480 2024-11-13T05:41:04,907 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe3d0d37041d47ea97a5e60c30cda08b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731476462020 2024-11-13T05:41:04,907 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 046bfdb5466b472f9b99b755d0bd48a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731476462020 2024-11-13T05:41:04,907 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f12c178a6fa4ddfaa1a1df26883a809, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731476463233 2024-11-13T05:41:04,907 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7338a32746c4332b496802f4bbdf6ca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731476463233 2024-11-13T05:41:04,915 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:04,916 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/154cf4cf4fc6459e84dc9fb0a6520157 is 50, key is test_row_0/B:col10/1731476463242/Put/seqid=0 2024-11-13T05:41:04,917 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#A#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:04,917 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/9c0c852414ea45278e593cc7d9d84133 is 50, key is test_row_0/A:col10/1731476463242/Put/seqid=0 2024-11-13T05:41:04,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742138_1314 (size=13051) 2024-11-13T05:41:04,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742139_1315 (size=13051) 2024-11-13T05:41:04,954 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/9c0c852414ea45278e593cc7d9d84133 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/9c0c852414ea45278e593cc7d9d84133 2024-11-13T05:41:04,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T05:41:04,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,956 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T05:41:04,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/56cc3f4d4de44e2aa3166960b3d222bf is 50, key is test_row_0/A:col10/1731476463737/Put/seqid=0 2024-11-13T05:41:04,964 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/A of f6767d6df0e08ef8c3ac8d792634f477 into 9c0c852414ea45278e593cc7d9d84133(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:04,964 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:04,966 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/A, priority=13, startTime=1731476464904; duration=0sec 2024-11-13T05:41:04,966 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:04,966 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:A 2024-11-13T05:41:04,966 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:04,968 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:04,968 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/C is initiating minor compaction (all files) 2024-11-13T05:41:04,968 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/C in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:04,968 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/9cc178f1743f4bc8b4819ec6d9c70609, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/4b722b8bac2d4abc8909cdf3eb821e24, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/81fe53eba0e441ab943a41d64f34e181] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=36.7 K 2024-11-13T05:41:04,969 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cc178f1743f4bc8b4819ec6d9c70609, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731476461480 2024-11-13T05:41:04,970 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b722b8bac2d4abc8909cdf3eb821e24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731476462020 2024-11-13T05:41:04,970 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81fe53eba0e441ab943a41d64f34e181, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731476463233 2024-11-13T05:41:04,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742140_1316 (size=12301) 2024-11-13T05:41:04,975 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/56cc3f4d4de44e2aa3166960b3d222bf 2024-11-13T05:41:04,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/fea88e1e78564057aea091a2dad62d30 is 50, key is test_row_0/B:col10/1731476463737/Put/seqid=0 2024-11-13T05:41:04,988 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#C#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:04,989 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/386525380e3240f8ada91c787423ea80 is 50, key is test_row_0/C:col10/1731476463242/Put/seqid=0 2024-11-13T05:41:05,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742141_1317 (size=12301) 2024-11-13T05:41:05,009 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/fea88e1e78564057aea091a2dad62d30 2024-11-13T05:41:05,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742142_1318 (size=13051) 2024-11-13T05:41:05,031 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/386525380e3240f8ada91c787423ea80 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/386525380e3240f8ada91c787423ea80 2024-11-13T05:41:05,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/68e76553137c491da7d48e374d24b24b is 50, key is test_row_0/C:col10/1731476463737/Put/seqid=0 2024-11-13T05:41:05,037 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/C of f6767d6df0e08ef8c3ac8d792634f477 into 386525380e3240f8ada91c787423ea80(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:05,037 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:05,037 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/C, priority=13, startTime=1731476464905; duration=0sec 2024-11-13T05:41:05,037 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:05,037 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:C 2024-11-13T05:41:05,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742143_1319 (size=12301) 2024-11-13T05:41:05,043 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/68e76553137c491da7d48e374d24b24b 2024-11-13T05:41:05,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/56cc3f4d4de44e2aa3166960b3d222bf as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/56cc3f4d4de44e2aa3166960b3d222bf 2024-11-13T05:41:05,052 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/56cc3f4d4de44e2aa3166960b3d222bf, entries=150, sequenceid=345, filesize=12.0 K 2024-11-13T05:41:05,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/fea88e1e78564057aea091a2dad62d30 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/fea88e1e78564057aea091a2dad62d30 2024-11-13T05:41:05,056 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/fea88e1e78564057aea091a2dad62d30, entries=150, sequenceid=345, filesize=12.0 K 2024-11-13T05:41:05,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/68e76553137c491da7d48e374d24b24b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/68e76553137c491da7d48e374d24b24b 2024-11-13T05:41:05,062 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/68e76553137c491da7d48e374d24b24b, entries=150, sequenceid=345, filesize=12.0 K 2024-11-13T05:41:05,063 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for f6767d6df0e08ef8c3ac8d792634f477 in 107ms, sequenceid=345, compaction requested=false 2024-11-13T05:41:05,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:05,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:05,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-13T05:41:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-13T05:41:05,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-13T05:41:05,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6410 sec 2024-11-13T05:41:05,069 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6450 sec 2024-11-13T05:41:05,342 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/154cf4cf4fc6459e84dc9fb0a6520157 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/154cf4cf4fc6459e84dc9fb0a6520157 2024-11-13T05:41:05,347 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/B of f6767d6df0e08ef8c3ac8d792634f477 into 154cf4cf4fc6459e84dc9fb0a6520157(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:05,347 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:05,347 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/B, priority=13, startTime=1731476464905; duration=0sec 2024-11-13T05:41:05,347 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:05,347 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:B 2024-11-13T05:41:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:41:05,424 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:05,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:41:05,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/4b594155276946039d375cdc8601e465 is 50, key is test_row_0/A:col10/1731476465420/Put/seqid=0 2024-11-13T05:41:05,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742144_1320 (size=14741) 2024-11-13T05:41:05,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/4b594155276946039d375cdc8601e465 2024-11-13T05:41:05,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/b8fbbfa356034e0088578c6ebed0276c is 50, key is test_row_0/B:col10/1731476465420/Put/seqid=0 2024-11-13T05:41:05,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742145_1321 (size=12301) 2024-11-13T05:41:05,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/b8fbbfa356034e0088578c6ebed0276c 2024-11-13T05:41:05,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/f2f0d9615b4f4ddd90096b10981a530c is 50, key is test_row_0/C:col10/1731476465420/Put/seqid=0 2024-11-13T05:41:05,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742146_1322 (size=12301) 2024-11-13T05:41:05,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476525506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:05,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:05,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:05,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T05:41:05,566 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:05,567 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-13T05:41:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T05:41:05,569 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:05,570 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:05,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:05,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476525616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:05,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44274 deadline: 1731476525647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:05,649 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:05,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44316 deadline: 1731476525656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:05,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44262 deadline: 1731476525656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:05,658 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:05,658 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in f6767d6df0e08ef8c3ac8d792634f477 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T05:41:05,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T05:41:05,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:05,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:05,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:05,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:05,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:05,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476525826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:05,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/f2f0d9615b4f4ddd90096b10981a530c 2024-11-13T05:41:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/4b594155276946039d375cdc8601e465 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/4b594155276946039d375cdc8601e465 2024-11-13T05:41:05,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/4b594155276946039d375cdc8601e465, entries=200, sequenceid=359, filesize=14.4 K 2024-11-13T05:41:05,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/b8fbbfa356034e0088578c6ebed0276c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/b8fbbfa356034e0088578c6ebed0276c 2024-11-13T05:41:05,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/b8fbbfa356034e0088578c6ebed0276c, entries=150, sequenceid=359, filesize=12.0 K 2024-11-13T05:41:05,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/f2f0d9615b4f4ddd90096b10981a530c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f2f0d9615b4f4ddd90096b10981a530c 2024-11-13T05:41:05,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f2f0d9615b4f4ddd90096b10981a530c, entries=150, sequenceid=359, filesize=12.0 K 2024-11-13T05:41:05,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f6767d6df0e08ef8c3ac8d792634f477 in 451ms, sequenceid=359, compaction requested=true 2024-11-13T05:41:05,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:05,875 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:05,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:05,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:05,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T05:41:05,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:05,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:05,875 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:05,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:05,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:05,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:05,876 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T05:41:05,876 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:05,876 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:05,876 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/B is initiating minor compaction (all files) 2024-11-13T05:41:05,876 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/A is initiating minor compaction (all files) 2024-11-13T05:41:05,876 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/A in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:05,876 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/B in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:05,876 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/154cf4cf4fc6459e84dc9fb0a6520157, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/fea88e1e78564057aea091a2dad62d30, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/b8fbbfa356034e0088578c6ebed0276c] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=36.8 K 2024-11-13T05:41:05,876 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/9c0c852414ea45278e593cc7d9d84133, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/56cc3f4d4de44e2aa3166960b3d222bf, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/4b594155276946039d375cdc8601e465] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=39.2 K 2024-11-13T05:41:05,877 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 154cf4cf4fc6459e84dc9fb0a6520157, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731476463233 2024-11-13T05:41:05,877 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c0c852414ea45278e593cc7d9d84133, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731476463233 2024-11-13T05:41:05,877 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting fea88e1e78564057aea091a2dad62d30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731476463715 2024-11-13T05:41:05,877 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56cc3f4d4de44e2aa3166960b3d222bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731476463715 2024-11-13T05:41:05,877 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting b8fbbfa356034e0088578c6ebed0276c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731476465409 2024-11-13T05:41:05,877 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b594155276946039d375cdc8601e465, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731476465409 2024-11-13T05:41:05,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/25365ba5cdf7452bb24a8f24627254ed is 50, key is test_row_0/A:col10/1731476465495/Put/seqid=0 2024-11-13T05:41:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T05:41:05,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742147_1323 (size=12301) 2024-11-13T05:41:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:41:05,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. as already flushing 2024-11-13T05:41:05,889 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/25365ba5cdf7452bb24a8f24627254ed 2024-11-13T05:41:05,892 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:05,893 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/076fef72970f4d0e8dcf4e635b09d3d5 is 50, key is test_row_0/B:col10/1731476465420/Put/seqid=0 2024-11-13T05:41:05,897 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#A#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:05,897 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/e4816d8656794286a5187b6d5835e406 is 50, key is test_row_0/A:col10/1731476465420/Put/seqid=0 2024-11-13T05:41:05,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/366226fc50b04ac592e8deebd68fc8b8 is 50, key is test_row_0/B:col10/1731476465495/Put/seqid=0 2024-11-13T05:41:05,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742149_1325 (size=13153) 2024-11-13T05:41:05,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742148_1324 (size=13153) 2024-11-13T05:41:05,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742150_1326 (size=12301) 2024-11-13T05:41:05,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476525922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:06,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476526036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:06,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476526136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:06,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:06,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:06,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T05:41:06,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476526246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:06,323 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/e4816d8656794286a5187b6d5835e406 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/e4816d8656794286a5187b6d5835e406 2024-11-13T05:41:06,324 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/076fef72970f4d0e8dcf4e635b09d3d5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/076fef72970f4d0e8dcf4e635b09d3d5 2024-11-13T05:41:06,324 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/366226fc50b04ac592e8deebd68fc8b8 2024-11-13T05:41:06,331 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/B of f6767d6df0e08ef8c3ac8d792634f477 into 076fef72970f4d0e8dcf4e635b09d3d5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:06,331 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:06,331 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/B, priority=13, startTime=1731476465875; duration=0sec 2024-11-13T05:41:06,331 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:06,331 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:B 2024-11-13T05:41:06,331 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:06,332 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:06,332 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/C is initiating minor compaction (all files) 2024-11-13T05:41:06,333 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/C in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:06,333 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/386525380e3240f8ada91c787423ea80, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/68e76553137c491da7d48e374d24b24b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f2f0d9615b4f4ddd90096b10981a530c] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=36.8 K 2024-11-13T05:41:06,333 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 386525380e3240f8ada91c787423ea80, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731476463233 2024-11-13T05:41:06,333 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 68e76553137c491da7d48e374d24b24b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731476463715 2024-11-13T05:41:06,334 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/A of f6767d6df0e08ef8c3ac8d792634f477 into e4816d8656794286a5187b6d5835e406(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:06,334 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:06,334 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/A, priority=13, startTime=1731476465875; duration=0sec 2024-11-13T05:41:06,334 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:06,334 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:A 2024-11-13T05:41:06,334 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f0d9615b4f4ddd90096b10981a530c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731476465409 2024-11-13T05:41:06,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/0a4339e756604c7e953add30c3f69bd6 is 50, key is test_row_0/C:col10/1731476465495/Put/seqid=0 2024-11-13T05:41:06,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742151_1327 (size=12301) 2024-11-13T05:41:06,356 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#C#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:06,357 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/3109e7f35d1446a6a529694b94bbcc26 is 50, key is test_row_0/C:col10/1731476465420/Put/seqid=0 2024-11-13T05:41:06,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742152_1328 (size=13153) 2024-11-13T05:41:06,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44312 deadline: 1731476526556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:06,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44286 deadline: 1731476526646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:06,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:06,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:06,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T05:41:06,753 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/0a4339e756604c7e953add30c3f69bd6 2024-11-13T05:41:06,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/25365ba5cdf7452bb24a8f24627254ed as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/25365ba5cdf7452bb24a8f24627254ed 2024-11-13T05:41:06,760 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/25365ba5cdf7452bb24a8f24627254ed, entries=150, sequenceid=384, filesize=12.0 K 2024-11-13T05:41:06,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/366226fc50b04ac592e8deebd68fc8b8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/366226fc50b04ac592e8deebd68fc8b8 2024-11-13T05:41:06,765 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/366226fc50b04ac592e8deebd68fc8b8, entries=150, sequenceid=384, filesize=12.0 K 2024-11-13T05:41:06,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/0a4339e756604c7e953add30c3f69bd6 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/0a4339e756604c7e953add30c3f69bd6 2024-11-13T05:41:06,769 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/0a4339e756604c7e953add30c3f69bd6, entries=150, sequenceid=384, filesize=12.0 K 2024-11-13T05:41:06,770 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f6767d6df0e08ef8c3ac8d792634f477 in 895ms, sequenceid=384, compaction requested=false 2024-11-13T05:41:06,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:06,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:06,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-13T05:41:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-13T05:41:06,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-13T05:41:06,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2010 sec 2024-11-13T05:41:06,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.2080 sec 2024-11-13T05:41:06,777 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/3109e7f35d1446a6a529694b94bbcc26 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/3109e7f35d1446a6a529694b94bbcc26 2024-11-13T05:41:06,782 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/C of f6767d6df0e08ef8c3ac8d792634f477 into 3109e7f35d1446a6a529694b94bbcc26(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:06,782 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:06,782 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/C, priority=13, startTime=1731476465875; duration=0sec 2024-11-13T05:41:06,782 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:06,782 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:C 2024-11-13T05:41:06,874 INFO [Thread-1165 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1165. 2024-11-13T05:41:06,874 DEBUG [Thread-1165 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:06,874 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:06,874 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:06,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:06,875 INFO [Thread-1163 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1163. 2024-11-13T05:41:06,875 DEBUG [Thread-1163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:06,875 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:06,875 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:06,875 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:06,876 INFO [Thread-1159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1159. 2024-11-13T05:41:06,876 INFO [Thread-1161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1161. 2024-11-13T05:41:06,876 DEBUG [Thread-1159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:06,876 DEBUG [Thread-1161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:06,876 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:06,876 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:06,876 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:06,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:06,876 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:06,876 INFO [Thread-1157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1157. 2024-11-13T05:41:06,876 DEBUG [Thread-1157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:06,876 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:06,876 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:06,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:06,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:41:07,068 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:07,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T05:41:07,068 INFO [Thread-1154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1154. 2024-11-13T05:41:07,068 DEBUG [Thread-1154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:07,068 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:07,069 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:07,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:07,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/a75a4dc5faac4091b3743ba8783efaab is 50, key is test_row_0/A:col10/1731476467066/Put/seqid=0 2024-11-13T05:41:07,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742153_1329 (size=9857) 2024-11-13T05:41:07,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/a75a4dc5faac4091b3743ba8783efaab 2024-11-13T05:41:07,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/db747c6979194a00a539069b5c6c46dd is 50, key is test_row_0/B:col10/1731476467066/Put/seqid=0 2024-11-13T05:41:07,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742154_1330 (size=9857) 2024-11-13T05:41:07,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/db747c6979194a00a539069b5c6c46dd 2024-11-13T05:41:07,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/d974c294bce746109a63bfa1b7d8181c is 50, key is test_row_0/C:col10/1731476467066/Put/seqid=0 2024-11-13T05:41:07,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742155_1331 (size=9857) 2024-11-13T05:41:07,668 INFO [Thread-1146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1146. 2024-11-13T05:41:07,668 DEBUG [Thread-1146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:07,668 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:07,668 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:07,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:07,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T05:41:07,717 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:07,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/d974c294bce746109a63bfa1b7d8181c 2024-11-13T05:41:07,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/a75a4dc5faac4091b3743ba8783efaab as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/a75a4dc5faac4091b3743ba8783efaab 2024-11-13T05:41:07,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/a75a4dc5faac4091b3743ba8783efaab, entries=100, sequenceid=399, filesize=9.6 K 2024-11-13T05:41:07,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/db747c6979194a00a539069b5c6c46dd as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/db747c6979194a00a539069b5c6c46dd 2024-11-13T05:41:07,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/db747c6979194a00a539069b5c6c46dd, entries=100, sequenceid=399, filesize=9.6 K 2024-11-13T05:41:07,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/d974c294bce746109a63bfa1b7d8181c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/d974c294bce746109a63bfa1b7d8181c 2024-11-13T05:41:07,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/d974c294bce746109a63bfa1b7d8181c, entries=100, sequenceid=399, filesize=9.6 K 2024-11-13T05:41:07,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=6.71 KB/6870 for f6767d6df0e08ef8c3ac8d792634f477 in 862ms, sequenceid=399, compaction requested=true 2024-11-13T05:41:07,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:07,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:07,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:07,931 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:07,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:07,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:07,931 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:07,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6767d6df0e08ef8c3ac8d792634f477:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:07,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:07,932 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:07,932 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/B is initiating minor compaction (all files) 2024-11-13T05:41:07,932 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/B in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:07,932 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/076fef72970f4d0e8dcf4e635b09d3d5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/366226fc50b04ac592e8deebd68fc8b8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/db747c6979194a00a539069b5c6c46dd] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=34.5 K 2024-11-13T05:41:07,932 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:07,932 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/A is initiating minor compaction (all files) 2024-11-13T05:41:07,932 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/A in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:07,933 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/e4816d8656794286a5187b6d5835e406, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/25365ba5cdf7452bb24a8f24627254ed, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/a75a4dc5faac4091b3743ba8783efaab] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=34.5 K 2024-11-13T05:41:07,933 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4816d8656794286a5187b6d5835e406, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731476465409 2024-11-13T05:41:07,933 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 076fef72970f4d0e8dcf4e635b09d3d5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731476465409 2024-11-13T05:41:07,933 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25365ba5cdf7452bb24a8f24627254ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731476465484 2024-11-13T05:41:07,933 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 366226fc50b04ac592e8deebd68fc8b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731476465484 2024-11-13T05:41:07,934 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting db747c6979194a00a539069b5c6c46dd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731476465921 2024-11-13T05:41:07,934 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting a75a4dc5faac4091b3743ba8783efaab, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731476465921 2024-11-13T05:41:07,943 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#A#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:07,944 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/5959ee6607ba4ca0ab2751ff9dd574d2 is 50, key is test_row_0/A:col10/1731476467066/Put/seqid=0 2024-11-13T05:41:07,952 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#B#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:07,952 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/8dc3d6213256412e8fce02ad9cafa4f3 is 50, key is test_row_0/B:col10/1731476467066/Put/seqid=0 2024-11-13T05:41:07,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742156_1332 (size=13255) 2024-11-13T05:41:07,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742157_1333 (size=13255) 2024-11-13T05:41:08,361 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/5959ee6607ba4ca0ab2751ff9dd574d2 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/5959ee6607ba4ca0ab2751ff9dd574d2 2024-11-13T05:41:08,374 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/A of f6767d6df0e08ef8c3ac8d792634f477 into 5959ee6607ba4ca0ab2751ff9dd574d2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:08,374 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:08,374 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/A, priority=13, startTime=1731476467931; duration=0sec 2024-11-13T05:41:08,374 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:08,374 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:A 2024-11-13T05:41:08,375 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:08,375 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/8dc3d6213256412e8fce02ad9cafa4f3 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/8dc3d6213256412e8fce02ad9cafa4f3 2024-11-13T05:41:08,376 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:08,376 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): f6767d6df0e08ef8c3ac8d792634f477/C is initiating minor compaction (all files) 2024-11-13T05:41:08,376 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6767d6df0e08ef8c3ac8d792634f477/C in TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:08,376 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/3109e7f35d1446a6a529694b94bbcc26, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/0a4339e756604c7e953add30c3f69bd6, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/d974c294bce746109a63bfa1b7d8181c] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp, totalSize=34.5 K 2024-11-13T05:41:08,378 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3109e7f35d1446a6a529694b94bbcc26, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731476465409 2024-11-13T05:41:08,378 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a4339e756604c7e953add30c3f69bd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731476465484 2024-11-13T05:41:08,379 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting d974c294bce746109a63bfa1b7d8181c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731476465921 2024-11-13T05:41:08,381 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/B of f6767d6df0e08ef8c3ac8d792634f477 into 8dc3d6213256412e8fce02ad9cafa4f3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:08,381 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:08,381 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/B, priority=13, startTime=1731476467931; duration=0sec 2024-11-13T05:41:08,381 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:08,381 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:B 2024-11-13T05:41:08,388 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6767d6df0e08ef8c3ac8d792634f477#C#compaction#287 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:08,388 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/f18c57bd90f34efba79bd1431845b971 is 50, key is test_row_0/C:col10/1731476467066/Put/seqid=0 2024-11-13T05:41:08,392 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:41:08,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742158_1334 (size=13255) 2024-11-13T05:41:08,802 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/f18c57bd90f34efba79bd1431845b971 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f18c57bd90f34efba79bd1431845b971 2024-11-13T05:41:08,810 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6767d6df0e08ef8c3ac8d792634f477/C of f6767d6df0e08ef8c3ac8d792634f477 into f18c57bd90f34efba79bd1431845b971(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:08,810 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6767d6df0e08ef8c3ac8d792634f477: 2024-11-13T05:41:08,810 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., storeName=f6767d6df0e08ef8c3ac8d792634f477/C, priority=13, startTime=1731476467931; duration=0sec 2024-11-13T05:41:08,810 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:08,811 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6767d6df0e08ef8c3ac8d792634f477:C 2024-11-13T05:41:09,682 INFO [Thread-1150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1150. 2024-11-13T05:41:09,682 DEBUG [Thread-1150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:09,682 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:09,682 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:09,682 INFO [Thread-1148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1148. 2024-11-13T05:41:09,682 INFO [Thread-1152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1152. 2024-11-13T05:41:09,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:09,682 DEBUG [Thread-1148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:09,682 DEBUG [Thread-1152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:09,682 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:09,682 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:09,682 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:09,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:09,682 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:09,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:09,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T05:41:09,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 183 2024-11-13T05:41:09,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-13T05:41:09,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-13T05:41:09,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-13T05:41:09,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-13T05:41:09,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T05:41:09,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3954 2024-11-13T05:41:09,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3945 2024-11-13T05:41:09,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3725 2024-11-13T05:41:09,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3959 2024-11-13T05:41:09,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3968 2024-11-13T05:41:09,683 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T05:41:09,683 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:41:09,683 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:41:09,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:09,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:09,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:09,684 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T05:41:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T05:41:09,687 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476469687"}]},"ts":"1731476469687"} 2024-11-13T05:41:09,688 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T05:41:09,688 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T05:41:09,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:41:09,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6767d6df0e08ef8c3ac8d792634f477, UNASSIGN}] 2024-11-13T05:41:09,691 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6767d6df0e08ef8c3ac8d792634f477, UNASSIGN 2024-11-13T05:41:09,691 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=f6767d6df0e08ef8c3ac8d792634f477, regionState=CLOSING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:09,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6767d6df0e08ef8c3ac8d792634f477, UNASSIGN because future has completed 2024-11-13T05:41:09,693 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:41:09,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; CloseRegionProcedure f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:41:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T05:41:09,845 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(122): Close f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:41:09,845 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:41:09,845 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1722): Closing f6767d6df0e08ef8c3ac8d792634f477, disabling compactions & flushes 2024-11-13T05:41:09,845 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:09,845 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:09,845 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. after waiting 0 ms 2024-11-13T05:41:09,845 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:09,845 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(2902): Flushing f6767d6df0e08ef8c3ac8d792634f477 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-13T05:41:09,850 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/2793c35ca98f46a09a301b4fb1db9bf8 is 50, key is test_row_0/A:col10/1731476467667/Put/seqid=0 2024-11-13T05:41:09,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742159_1335 (size=12301) 2024-11-13T05:41:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T05:41:10,256 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/2793c35ca98f46a09a301b4fb1db9bf8 2024-11-13T05:41:10,268 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/4bbae75a67d94a4bb6454f961150d117 is 50, key is test_row_0/B:col10/1731476467667/Put/seqid=0 2024-11-13T05:41:10,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742160_1336 (size=12301) 2024-11-13T05:41:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T05:41:10,674 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/4bbae75a67d94a4bb6454f961150d117 2024-11-13T05:41:10,691 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/95d11cc1754447cb979d2fa431bd72ff is 50, key is test_row_0/C:col10/1731476467667/Put/seqid=0 2024-11-13T05:41:10,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742161_1337 (size=12301) 2024-11-13T05:41:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T05:41:11,096 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/95d11cc1754447cb979d2fa431bd72ff 2024-11-13T05:41:11,106 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/A/2793c35ca98f46a09a301b4fb1db9bf8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/2793c35ca98f46a09a301b4fb1db9bf8 2024-11-13T05:41:11,111 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/2793c35ca98f46a09a301b4fb1db9bf8, entries=150, sequenceid=409, filesize=12.0 K 2024-11-13T05:41:11,112 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/B/4bbae75a67d94a4bb6454f961150d117 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4bbae75a67d94a4bb6454f961150d117 2024-11-13T05:41:11,116 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4bbae75a67d94a4bb6454f961150d117, entries=150, sequenceid=409, filesize=12.0 K 2024-11-13T05:41:11,117 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/.tmp/C/95d11cc1754447cb979d2fa431bd72ff as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/95d11cc1754447cb979d2fa431bd72ff 2024-11-13T05:41:11,121 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/95d11cc1754447cb979d2fa431bd72ff, entries=150, sequenceid=409, filesize=12.0 K 2024-11-13T05:41:11,121 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f6767d6df0e08ef8c3ac8d792634f477 in 1276ms, sequenceid=409, compaction requested=false 2024-11-13T05:41:11,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/e8833732ec5e432a9bd839288cc2bb1c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/a8dff56bc3174f20b8513132901e3f1c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/609493020bf44bbb95f23737369830c0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/842305cb3bf742ad8ec71e063fea7036, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/640c7becab074e5ca740bec7db9c1e73, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/85b119b248354f688b00e664fda6db6e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/c7710521427c47c39868cdef7a01851b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/5f8d0af2e41a439b91ef2a9f91cb2b93, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/ea7d22302c734de38aee83cf90ee2252, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/05e0696c6f6a486cacc039c379cd3a6c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/760ddb218f2e4d389a88ffdf02039b1c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/f05de721aefe4d87b42ce54c152c0b96, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/d311864b246b406683a20f786d3a38e0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/0dd3f28cff9a40909f9a630ab361f80d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/75a084f10b0e4555a7c4d863c479c88e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/56c40f8fd5c8442fbbbb12760b21cb04, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/02dcba5914864d7095e438f747b79fe4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/4767e63eaf4e4a5b9c9034e965339629, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/587612f52ea94de5a455c7c027afa279, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/9e8e22ccefdb4351b2d2f1b1fae8499e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/fe3d0d37041d47ea97a5e60c30cda08b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/c7338a32746c4332b496802f4bbdf6ca, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/9c0c852414ea45278e593cc7d9d84133, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/56cc3f4d4de44e2aa3166960b3d222bf, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/4b594155276946039d375cdc8601e465, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/e4816d8656794286a5187b6d5835e406, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/25365ba5cdf7452bb24a8f24627254ed, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/a75a4dc5faac4091b3743ba8783efaab] to archive 2024-11-13T05:41:11,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:41:11,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/e8833732ec5e432a9bd839288cc2bb1c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/e8833732ec5e432a9bd839288cc2bb1c 2024-11-13T05:41:11,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/a8dff56bc3174f20b8513132901e3f1c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/a8dff56bc3174f20b8513132901e3f1c 2024-11-13T05:41:11,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/609493020bf44bbb95f23737369830c0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/609493020bf44bbb95f23737369830c0 2024-11-13T05:41:11,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/842305cb3bf742ad8ec71e063fea7036 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/842305cb3bf742ad8ec71e063fea7036 2024-11-13T05:41:11,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/640c7becab074e5ca740bec7db9c1e73 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/640c7becab074e5ca740bec7db9c1e73 2024-11-13T05:41:11,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/85b119b248354f688b00e664fda6db6e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/85b119b248354f688b00e664fda6db6e 2024-11-13T05:41:11,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/c7710521427c47c39868cdef7a01851b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/c7710521427c47c39868cdef7a01851b 2024-11-13T05:41:11,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/5f8d0af2e41a439b91ef2a9f91cb2b93 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/5f8d0af2e41a439b91ef2a9f91cb2b93 2024-11-13T05:41:11,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/ea7d22302c734de38aee83cf90ee2252 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/ea7d22302c734de38aee83cf90ee2252 2024-11-13T05:41:11,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/05e0696c6f6a486cacc039c379cd3a6c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/05e0696c6f6a486cacc039c379cd3a6c 2024-11-13T05:41:11,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/760ddb218f2e4d389a88ffdf02039b1c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/760ddb218f2e4d389a88ffdf02039b1c 2024-11-13T05:41:11,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/f05de721aefe4d87b42ce54c152c0b96 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/f05de721aefe4d87b42ce54c152c0b96 2024-11-13T05:41:11,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/d311864b246b406683a20f786d3a38e0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/d311864b246b406683a20f786d3a38e0 2024-11-13T05:41:11,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/0dd3f28cff9a40909f9a630ab361f80d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/0dd3f28cff9a40909f9a630ab361f80d 2024-11-13T05:41:11,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/75a084f10b0e4555a7c4d863c479c88e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/75a084f10b0e4555a7c4d863c479c88e 2024-11-13T05:41:11,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/56c40f8fd5c8442fbbbb12760b21cb04 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/56c40f8fd5c8442fbbbb12760b21cb04 2024-11-13T05:41:11,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/02dcba5914864d7095e438f747b79fe4 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/02dcba5914864d7095e438f747b79fe4 2024-11-13T05:41:11,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/4767e63eaf4e4a5b9c9034e965339629 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/4767e63eaf4e4a5b9c9034e965339629 2024-11-13T05:41:11,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/587612f52ea94de5a455c7c027afa279 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/587612f52ea94de5a455c7c027afa279 2024-11-13T05:41:11,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/9e8e22ccefdb4351b2d2f1b1fae8499e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/9e8e22ccefdb4351b2d2f1b1fae8499e 2024-11-13T05:41:11,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/fe3d0d37041d47ea97a5e60c30cda08b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/fe3d0d37041d47ea97a5e60c30cda08b 2024-11-13T05:41:11,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/c7338a32746c4332b496802f4bbdf6ca to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/c7338a32746c4332b496802f4bbdf6ca 2024-11-13T05:41:11,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/9c0c852414ea45278e593cc7d9d84133 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/9c0c852414ea45278e593cc7d9d84133 2024-11-13T05:41:11,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/56cc3f4d4de44e2aa3166960b3d222bf to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/56cc3f4d4de44e2aa3166960b3d222bf 2024-11-13T05:41:11,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/4b594155276946039d375cdc8601e465 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/4b594155276946039d375cdc8601e465 2024-11-13T05:41:11,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/e4816d8656794286a5187b6d5835e406 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/e4816d8656794286a5187b6d5835e406 2024-11-13T05:41:11,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/25365ba5cdf7452bb24a8f24627254ed to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/25365ba5cdf7452bb24a8f24627254ed 2024-11-13T05:41:11,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/a75a4dc5faac4091b3743ba8783efaab to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/a75a4dc5faac4091b3743ba8783efaab 2024-11-13T05:41:11,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dae0ea5aa60b4e6482c9f1e1d8249c0e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dded9f13da9a4d07a19f704ae9fc9c60, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/b8e7c78817344821aaf5330b9ecbcc22, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dc591ac8c54344eda0edf8daa8a4d229, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/a7c1383efda442fc82aeba72132c8bad, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/fee1d58419fc4f468eb22ac85d8bae92, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/d9b6f41219d64997bfbab5d340c3206a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/633cb3ff3a5f4757a6fdfb76b8db1280, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/33519e7408e04ea4913eb1dd0bc39536, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/ca5fa28d5ed44fa39039dc4f3f944e84, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/9aa7eb89e7d543e4943a2000c7622194, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/9bded2abe76d4d1e9fe328e909dd7557, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/5d65cd0ee3e24fda93c5f7d86838dc55, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/94102181fba74248a9a092b80daf9eb7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/81e2b61fa06a41f88c342c767254e9f5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/f419204b8c1044298b72d4055a598597, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/389713e33b3b40afa43b120d8d6166e9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/bc06725c64274e15a83640133bd4eb2f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/46496f6983af46578e600abe9c3fbda4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4857bec2bde34b2287a859c11ca27b17, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/046bfdb5466b472f9b99b755d0bd48a0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/154cf4cf4fc6459e84dc9fb0a6520157, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4f12c178a6fa4ddfaa1a1df26883a809, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/fea88e1e78564057aea091a2dad62d30, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/076fef72970f4d0e8dcf4e635b09d3d5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/b8fbbfa356034e0088578c6ebed0276c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/366226fc50b04ac592e8deebd68fc8b8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/db747c6979194a00a539069b5c6c46dd] to archive 2024-11-13T05:41:11,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:41:11,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dae0ea5aa60b4e6482c9f1e1d8249c0e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dae0ea5aa60b4e6482c9f1e1d8249c0e 2024-11-13T05:41:11,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dded9f13da9a4d07a19f704ae9fc9c60 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dded9f13da9a4d07a19f704ae9fc9c60 2024-11-13T05:41:11,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/b8e7c78817344821aaf5330b9ecbcc22 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/b8e7c78817344821aaf5330b9ecbcc22 2024-11-13T05:41:11,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dc591ac8c54344eda0edf8daa8a4d229 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/dc591ac8c54344eda0edf8daa8a4d229 2024-11-13T05:41:11,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/a7c1383efda442fc82aeba72132c8bad to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/a7c1383efda442fc82aeba72132c8bad 2024-11-13T05:41:11,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/fee1d58419fc4f468eb22ac85d8bae92 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/fee1d58419fc4f468eb22ac85d8bae92 2024-11-13T05:41:11,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/d9b6f41219d64997bfbab5d340c3206a to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/d9b6f41219d64997bfbab5d340c3206a 2024-11-13T05:41:11,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/633cb3ff3a5f4757a6fdfb76b8db1280 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/633cb3ff3a5f4757a6fdfb76b8db1280 2024-11-13T05:41:11,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/33519e7408e04ea4913eb1dd0bc39536 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/33519e7408e04ea4913eb1dd0bc39536 2024-11-13T05:41:11,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/ca5fa28d5ed44fa39039dc4f3f944e84 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/ca5fa28d5ed44fa39039dc4f3f944e84 2024-11-13T05:41:11,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/9aa7eb89e7d543e4943a2000c7622194 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/9aa7eb89e7d543e4943a2000c7622194 2024-11-13T05:41:11,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/9bded2abe76d4d1e9fe328e909dd7557 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/9bded2abe76d4d1e9fe328e909dd7557 2024-11-13T05:41:11,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/5d65cd0ee3e24fda93c5f7d86838dc55 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/5d65cd0ee3e24fda93c5f7d86838dc55 2024-11-13T05:41:11,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/94102181fba74248a9a092b80daf9eb7 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/94102181fba74248a9a092b80daf9eb7 2024-11-13T05:41:11,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/81e2b61fa06a41f88c342c767254e9f5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/81e2b61fa06a41f88c342c767254e9f5 2024-11-13T05:41:11,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/f419204b8c1044298b72d4055a598597 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/f419204b8c1044298b72d4055a598597 2024-11-13T05:41:11,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/389713e33b3b40afa43b120d8d6166e9 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/389713e33b3b40afa43b120d8d6166e9 2024-11-13T05:41:11,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/bc06725c64274e15a83640133bd4eb2f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/bc06725c64274e15a83640133bd4eb2f 2024-11-13T05:41:11,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/46496f6983af46578e600abe9c3fbda4 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/46496f6983af46578e600abe9c3fbda4 2024-11-13T05:41:11,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4857bec2bde34b2287a859c11ca27b17 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4857bec2bde34b2287a859c11ca27b17 2024-11-13T05:41:11,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/046bfdb5466b472f9b99b755d0bd48a0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/046bfdb5466b472f9b99b755d0bd48a0 2024-11-13T05:41:11,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/154cf4cf4fc6459e84dc9fb0a6520157 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/154cf4cf4fc6459e84dc9fb0a6520157 2024-11-13T05:41:11,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4f12c178a6fa4ddfaa1a1df26883a809 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4f12c178a6fa4ddfaa1a1df26883a809 2024-11-13T05:41:11,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/fea88e1e78564057aea091a2dad62d30 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/fea88e1e78564057aea091a2dad62d30 2024-11-13T05:41:11,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/076fef72970f4d0e8dcf4e635b09d3d5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/076fef72970f4d0e8dcf4e635b09d3d5 2024-11-13T05:41:11,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/b8fbbfa356034e0088578c6ebed0276c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/b8fbbfa356034e0088578c6ebed0276c 2024-11-13T05:41:11,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/366226fc50b04ac592e8deebd68fc8b8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/366226fc50b04ac592e8deebd68fc8b8 2024-11-13T05:41:11,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/db747c6979194a00a539069b5c6c46dd to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/db747c6979194a00a539069b5c6c46dd 2024-11-13T05:41:11,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/88ae212ee09641b7b2150ef1600bca3e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/ee38c95bc2714faf9d5a31b570b98691, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/80c2f278222b4a83a29b8844496d7f25, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f2ac6e5e270048c0886ce45a325df0ca, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/56db065403c6407c839069a85c6567e4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/32339951055c44b490afd88fbf6c487b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/99d40833d4214f4885b6f5f204168079, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/309a63815d884bbab601fce03100f579, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/e56c686580ca4f4f9ea2be601e6fa213, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/a6fe23e6bc2648459a3d953e576b331f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/962e6346578b421da21db21120ccb883, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/19af0724016e43329e3ef3a6beca54c9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/8785ffb489624f0582d93dc8ed350316, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/6d5f4dd441e044b89012d8b0f45df374, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/81d92576a3054369875fc56bd03da1ea, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/30d8bdc94b7d49cc85e04ff1e96d4b42, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/fa16ad848d9d43359e0e5e2aad151951, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/851d15b4efb844d58c7b8c5fa172e4e2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/9cc178f1743f4bc8b4819ec6d9c70609, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/611f2f46bf164177b29ec39966a446c9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/4b722b8bac2d4abc8909cdf3eb821e24, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/386525380e3240f8ada91c787423ea80, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/81fe53eba0e441ab943a41d64f34e181, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/68e76553137c491da7d48e374d24b24b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/3109e7f35d1446a6a529694b94bbcc26, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f2f0d9615b4f4ddd90096b10981a530c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/0a4339e756604c7e953add30c3f69bd6, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/d974c294bce746109a63bfa1b7d8181c] to archive 2024-11-13T05:41:11,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:41:11,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/88ae212ee09641b7b2150ef1600bca3e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/88ae212ee09641b7b2150ef1600bca3e 2024-11-13T05:41:11,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/ee38c95bc2714faf9d5a31b570b98691 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/ee38c95bc2714faf9d5a31b570b98691 2024-11-13T05:41:11,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/80c2f278222b4a83a29b8844496d7f25 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/80c2f278222b4a83a29b8844496d7f25 2024-11-13T05:41:11,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f2ac6e5e270048c0886ce45a325df0ca to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f2ac6e5e270048c0886ce45a325df0ca 2024-11-13T05:41:11,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/56db065403c6407c839069a85c6567e4 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/56db065403c6407c839069a85c6567e4 2024-11-13T05:41:11,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/32339951055c44b490afd88fbf6c487b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/32339951055c44b490afd88fbf6c487b 2024-11-13T05:41:11,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/99d40833d4214f4885b6f5f204168079 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/99d40833d4214f4885b6f5f204168079 2024-11-13T05:41:11,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/309a63815d884bbab601fce03100f579 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/309a63815d884bbab601fce03100f579 2024-11-13T05:41:11,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/e56c686580ca4f4f9ea2be601e6fa213 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/e56c686580ca4f4f9ea2be601e6fa213 2024-11-13T05:41:11,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/a6fe23e6bc2648459a3d953e576b331f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/a6fe23e6bc2648459a3d953e576b331f 2024-11-13T05:41:11,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/962e6346578b421da21db21120ccb883 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/962e6346578b421da21db21120ccb883 2024-11-13T05:41:11,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/19af0724016e43329e3ef3a6beca54c9 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/19af0724016e43329e3ef3a6beca54c9 2024-11-13T05:41:11,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/8785ffb489624f0582d93dc8ed350316 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/8785ffb489624f0582d93dc8ed350316 2024-11-13T05:41:11,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/6d5f4dd441e044b89012d8b0f45df374 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/6d5f4dd441e044b89012d8b0f45df374 2024-11-13T05:41:11,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/81d92576a3054369875fc56bd03da1ea to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/81d92576a3054369875fc56bd03da1ea 2024-11-13T05:41:11,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/30d8bdc94b7d49cc85e04ff1e96d4b42 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/30d8bdc94b7d49cc85e04ff1e96d4b42 2024-11-13T05:41:11,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/fa16ad848d9d43359e0e5e2aad151951 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/fa16ad848d9d43359e0e5e2aad151951 2024-11-13T05:41:11,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/851d15b4efb844d58c7b8c5fa172e4e2 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/851d15b4efb844d58c7b8c5fa172e4e2 2024-11-13T05:41:11,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/9cc178f1743f4bc8b4819ec6d9c70609 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/9cc178f1743f4bc8b4819ec6d9c70609 2024-11-13T05:41:11,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/611f2f46bf164177b29ec39966a446c9 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/611f2f46bf164177b29ec39966a446c9 2024-11-13T05:41:11,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/4b722b8bac2d4abc8909cdf3eb821e24 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/4b722b8bac2d4abc8909cdf3eb821e24 2024-11-13T05:41:11,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/386525380e3240f8ada91c787423ea80 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/386525380e3240f8ada91c787423ea80 2024-11-13T05:41:11,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/81fe53eba0e441ab943a41d64f34e181 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/81fe53eba0e441ab943a41d64f34e181 2024-11-13T05:41:11,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/68e76553137c491da7d48e374d24b24b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/68e76553137c491da7d48e374d24b24b 2024-11-13T05:41:11,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/3109e7f35d1446a6a529694b94bbcc26 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/3109e7f35d1446a6a529694b94bbcc26 2024-11-13T05:41:11,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f2f0d9615b4f4ddd90096b10981a530c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f2f0d9615b4f4ddd90096b10981a530c 2024-11-13T05:41:11,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/0a4339e756604c7e953add30c3f69bd6 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/0a4339e756604c7e953add30c3f69bd6 2024-11-13T05:41:11,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/d974c294bce746109a63bfa1b7d8181c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/d974c294bce746109a63bfa1b7d8181c 2024-11-13T05:41:11,207 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/recovered.edits/412.seqid, newMaxSeqId=412, maxSeqId=1 2024-11-13T05:41:11,207 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. 2024-11-13T05:41:11,207 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1676): Region close journal for f6767d6df0e08ef8c3ac8d792634f477: Waiting for close lock at 1731476469845Running coprocessor pre-close hooks at 1731476469845Disabling compacts and flushes for region at 1731476469845Disabling writes for close at 1731476469845Obtaining lock to block concurrent updates at 1731476469845Preparing flush snapshotting stores in f6767d6df0e08ef8c3ac8d792634f477 at 1731476469845Finished memstore snapshotting TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731476469845Flushing stores of TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477. at 1731476469846 (+1 ms)Flushing f6767d6df0e08ef8c3ac8d792634f477/A: creating writer at 1731476469846Flushing f6767d6df0e08ef8c3ac8d792634f477/A: appending metadata at 1731476469849 (+3 ms)Flushing f6767d6df0e08ef8c3ac8d792634f477/A: closing flushed file at 1731476469849Flushing f6767d6df0e08ef8c3ac8d792634f477/B: creating writer at 1731476470265 (+416 ms)Flushing f6767d6df0e08ef8c3ac8d792634f477/B: appending metadata at 1731476470268 (+3 ms)Flushing f6767d6df0e08ef8c3ac8d792634f477/B: closing flushed file at 1731476470268Flushing f6767d6df0e08ef8c3ac8d792634f477/C: creating writer at 1731476470685 (+417 ms)Flushing f6767d6df0e08ef8c3ac8d792634f477/C: appending metadata at 1731476470690 (+5 ms)Flushing f6767d6df0e08ef8c3ac8d792634f477/C: closing flushed file at 1731476470690Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@be159e2: reopening flushed file at 1731476471106 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41d46536: reopening flushed file at 1731476471111 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c1969e8: reopening flushed file at 1731476471116 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f6767d6df0e08ef8c3ac8d792634f477 in 1276ms, sequenceid=409, compaction requested=false at 1731476471121 (+5 ms)Writing region close event to WAL at 1731476471204 (+83 ms)Running coprocessor post-close hooks at 1731476471207 (+3 ms)Closed at 1731476471207 2024-11-13T05:41:11,209 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(157): Closed f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:41:11,209 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=f6767d6df0e08ef8c3ac8d792634f477, regionState=CLOSED 2024-11-13T05:41:11,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=74, state=RUNNABLE, hasLock=false; CloseRegionProcedure f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:41:11,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-13T05:41:11,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseRegionProcedure f6767d6df0e08ef8c3ac8d792634f477, server=bc386b7bf2fe,45837,1731476380128 in 1.5180 sec 2024-11-13T05:41:11,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-11-13T05:41:11,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6767d6df0e08ef8c3ac8d792634f477, UNASSIGN in 1.5230 sec 2024-11-13T05:41:11,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-13T05:41:11,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5260 sec 2024-11-13T05:41:11,218 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476471217"}]},"ts":"1731476471217"} 2024-11-13T05:41:11,219 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T05:41:11,219 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T05:41:11,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5360 sec 2024-11-13T05:41:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T05:41:11,837 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:11,838 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T05:41:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:11,842 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=76, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T05:41:11,844 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=76, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:11,846 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:41:11,850 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/recovered.edits] 2024-11-13T05:41:11,854 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/2793c35ca98f46a09a301b4fb1db9bf8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/2793c35ca98f46a09a301b4fb1db9bf8 2024-11-13T05:41:11,856 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/5959ee6607ba4ca0ab2751ff9dd574d2 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/A/5959ee6607ba4ca0ab2751ff9dd574d2 2024-11-13T05:41:11,859 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4bbae75a67d94a4bb6454f961150d117 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/4bbae75a67d94a4bb6454f961150d117 2024-11-13T05:41:11,860 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/8dc3d6213256412e8fce02ad9cafa4f3 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/B/8dc3d6213256412e8fce02ad9cafa4f3 2024-11-13T05:41:11,861 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/95d11cc1754447cb979d2fa431bd72ff to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/95d11cc1754447cb979d2fa431bd72ff 2024-11-13T05:41:11,862 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f18c57bd90f34efba79bd1431845b971 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/C/f18c57bd90f34efba79bd1431845b971 2024-11-13T05:41:11,863 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/recovered.edits/412.seqid to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477/recovered.edits/412.seqid 2024-11-13T05:41:11,864 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/f6767d6df0e08ef8c3ac8d792634f477 2024-11-13T05:41:11,864 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T05:41:11,865 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=76, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:11,868 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T05:41:11,870 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T05:41:11,871 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=76, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:11,871 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T05:41:11,871 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731476471871"}]},"ts":"9223372036854775807"} 2024-11-13T05:41:11,873 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T05:41:11,873 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f6767d6df0e08ef8c3ac8d792634f477, NAME => 'TestAcidGuarantees,,1731476444676.f6767d6df0e08ef8c3ac8d792634f477.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T05:41:11,873 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T05:41:11,873 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731476471873"}]},"ts":"9223372036854775807"} 2024-11-13T05:41:11,875 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T05:41:11,875 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=76, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:11,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-11-13T05:41:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T05:41:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T05:41:11,947 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:11,959 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241 (was 240) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1793353738_22 at /127.0.0.1:39532 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-782295199_22 at /127.0.0.1:55320 [Waiting for operation #537] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-782295199_22 at /127.0.0.1:55244 [Waiting for operation #566] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=477 (was 539), ProcessCount=11 (was 11), AvailableMemoryMB=6831 (was 7266) 2024-11-13T05:41:11,967 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=477, ProcessCount=11, AvailableMemoryMB=6831 2024-11-13T05:41:11,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:41:11,968 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:41:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=77, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:11,970 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:41:11,970 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:11,970 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 77 2024-11-13T05:41:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-13T05:41:11,971 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:41:11,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742162_1338 (size=959) 2024-11-13T05:41:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-13T05:41:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-13T05:41:12,379 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745 2024-11-13T05:41:12,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742163_1339 (size=53) 2024-11-13T05:41:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-13T05:41:12,848 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:41:12,848 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cebd6d1afed0df6f58e14d0610d86767, disabling compactions & flushes 2024-11-13T05:41:12,848 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:12,848 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:12,848 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. after waiting 0 ms 2024-11-13T05:41:12,848 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:12,848 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:12,848 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cebd6d1afed0df6f58e14d0610d86767: Waiting for close lock at 1731476472848Disabling compacts and flushes for region at 1731476472848Disabling writes for close at 1731476472848Writing region close event to WAL at 1731476472848Closed at 1731476472848 2024-11-13T05:41:12,850 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:41:12,850 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731476472850"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731476472850"}]},"ts":"1731476472850"} 2024-11-13T05:41:12,852 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:41:12,852 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:41:12,853 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476472852"}]},"ts":"1731476472852"} 2024-11-13T05:41:12,854 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T05:41:12,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, ASSIGN}] 2024-11-13T05:41:12,856 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, ASSIGN 2024-11-13T05:41:12,856 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45837,1731476380128; forceNewPlan=false, retain=false 2024-11-13T05:41:13,007 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=cebd6d1afed0df6f58e14d0610d86767, regionState=OPENING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:13,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, ASSIGN because future has completed 2024-11-13T05:41:13,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:41:13,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-13T05:41:13,169 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:13,170 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:41:13,170 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:13,170 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:41:13,170 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:13,170 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:13,172 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:13,173 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cebd6d1afed0df6f58e14d0610d86767 columnFamilyName A 2024-11-13T05:41:13,174 DEBUG [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:13,174 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] regionserver.HStore(327): Store=cebd6d1afed0df6f58e14d0610d86767/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:41:13,174 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:13,176 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cebd6d1afed0df6f58e14d0610d86767 columnFamilyName B 2024-11-13T05:41:13,176 DEBUG [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:13,176 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] regionserver.HStore(327): Store=cebd6d1afed0df6f58e14d0610d86767/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:41:13,176 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:13,178 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cebd6d1afed0df6f58e14d0610d86767 columnFamilyName C 2024-11-13T05:41:13,178 DEBUG [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:13,178 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] regionserver.HStore(327): Store=cebd6d1afed0df6f58e14d0610d86767/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:41:13,178 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:13,179 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:13,180 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:13,181 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:13,181 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:13,182 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:41:13,183 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:13,185 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:41:13,186 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened cebd6d1afed0df6f58e14d0610d86767; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64248820, jitterRate=-0.042617976665496826}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:41:13,186 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:13,187 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for cebd6d1afed0df6f58e14d0610d86767: Running coprocessor pre-open hook at 1731476473170Writing region info on filesystem at 1731476473170Initializing all the Stores at 1731476473171 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476473171Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476473172 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476473172Cleaning up temporary data from old regions at 1731476473181 (+9 ms)Running coprocessor post-open hooks at 1731476473186 (+5 ms)Region opened successfully at 1731476473187 (+1 ms) 2024-11-13T05:41:13,187 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., pid=79, masterSystemTime=1731476473162 2024-11-13T05:41:13,189 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:13,189 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:13,190 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=cebd6d1afed0df6f58e14d0610d86767, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:13,191 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:41:13,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-13T05:41:13,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; OpenRegionProcedure cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 in 183 msec 2024-11-13T05:41:13,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-13T05:41:13,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, ASSIGN in 340 msec 2024-11-13T05:41:13,196 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:41:13,197 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476473196"}]},"ts":"1731476473196"} 2024-11-13T05:41:13,198 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T05:41:13,199 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:41:13,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2310 sec 2024-11-13T05:41:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-13T05:41:14,127 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:14,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ab9835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:14,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:14,131 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:14,133 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:14,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:14,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff16d77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:14,136 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64ac60cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:14,137 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:14,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:14,139 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:41:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,37913,1731476379290 2024-11-13T05:41:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5dead33b 2024-11-13T05:41:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:41:14,141 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:41:14,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:41:14,143 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:41:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:14,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742164_1340 (size=995) 2024-11-13T05:41:14,562 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-13T05:41:14,562 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-13T05:41:14,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:41:14,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, REOPEN/MOVE}] 2024-11-13T05:41:14,571 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, REOPEN/MOVE 2024-11-13T05:41:14,572 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=cebd6d1afed0df6f58e14d0610d86767, regionState=CLOSING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:14,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, REOPEN/MOVE because future has completed 2024-11-13T05:41:14,574 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:41:14,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:41:14,727 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:14,727 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:41:14,727 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing cebd6d1afed0df6f58e14d0610d86767, disabling compactions & flushes 2024-11-13T05:41:14,727 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:14,727 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:14,727 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. after waiting 0 ms 2024-11-13T05:41:14,727 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:14,734 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:41:14,735 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:14,735 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for cebd6d1afed0df6f58e14d0610d86767: Waiting for close lock at 1731476474727Running coprocessor pre-close hooks at 1731476474727Disabling compacts and flushes for region at 1731476474727Disabling writes for close at 1731476474727Writing region close event to WAL at 1731476474728 (+1 ms)Running coprocessor post-close hooks at 1731476474735 (+7 ms)Closed at 1731476474735 2024-11-13T05:41:14,735 WARN [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionServer(3299): Not adding moved region record: cebd6d1afed0df6f58e14d0610d86767 to self. 2024-11-13T05:41:14,739 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:14,739 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=cebd6d1afed0df6f58e14d0610d86767, regionState=CLOSED 2024-11-13T05:41:14,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:41:14,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-13T05:41:14,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 in 169 msec 2024-11-13T05:41:14,747 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, REOPEN/MOVE; state=CLOSED, location=bc386b7bf2fe,45837,1731476380128; forceNewPlan=false, retain=true 2024-11-13T05:41:14,899 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=cebd6d1afed0df6f58e14d0610d86767, regionState=OPENING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:14,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, REOPEN/MOVE because future has completed 2024-11-13T05:41:14,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:41:15,061 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:15,061 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7752): Opening region: {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:41:15,062 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,062 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:41:15,062 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7794): checking encryption for cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,062 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7797): checking classloading for cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,065 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,066 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cebd6d1afed0df6f58e14d0610d86767 columnFamilyName A 2024-11-13T05:41:15,068 DEBUG [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:15,068 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] regionserver.HStore(327): Store=cebd6d1afed0df6f58e14d0610d86767/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:41:15,069 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,070 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cebd6d1afed0df6f58e14d0610d86767 columnFamilyName B 2024-11-13T05:41:15,070 DEBUG [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:15,071 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] regionserver.HStore(327): Store=cebd6d1afed0df6f58e14d0610d86767/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:41:15,071 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,072 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cebd6d1afed0df6f58e14d0610d86767 columnFamilyName C 2024-11-13T05:41:15,073 DEBUG [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:15,073 INFO [StoreOpener-cebd6d1afed0df6f58e14d0610d86767-1 {}] regionserver.HStore(327): Store=cebd6d1afed0df6f58e14d0610d86767/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:41:15,074 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1038): replaying wal for cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,075 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,077 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,079 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1048): stopping wal replay for cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,079 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1060): Cleaning up temporary data for cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,080 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:41:15,082 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1093): writing seq id for cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,084 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1114): Opened cebd6d1afed0df6f58e14d0610d86767; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62256761, jitterRate=-0.0723019689321518}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:41:15,084 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,085 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1006): Region open journal for cebd6d1afed0df6f58e14d0610d86767: Running coprocessor pre-open hook at 1731476475062Writing region info on filesystem at 1731476475062Initializing all the Stores at 1731476475064 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476475064Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476475064Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476475064Cleaning up temporary data from old regions at 1731476475079 (+15 ms)Running coprocessor post-open hooks at 1731476475084 (+5 ms)Region opened successfully at 1731476475085 (+1 ms) 2024-11-13T05:41:15,086 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., pid=84, masterSystemTime=1731476475056 2024-11-13T05:41:15,088 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:15,088 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:15,089 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=cebd6d1afed0df6f58e14d0610d86767, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:41:15,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=82 2024-11-13T05:41:15,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 in 189 msec 2024-11-13T05:41:15,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-13T05:41:15,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, REOPEN/MOVE in 523 msec 2024-11-13T05:41:15,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-13T05:41:15,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-13T05:41:15,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 954 msec 2024-11-13T05:41:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T05:41:15,100 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:15,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11317fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:15,102 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:15,103 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af07ed3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:15,104 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76e95418, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,105 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:15,105 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:15,106 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@654a18a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:15,107 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:15,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c9f36a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,108 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,108 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:15,108 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:15,109 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:15,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:15,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:15,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae45edb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:15,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:15,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:15,110 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fb21c06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,111 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:15,111 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:15,112 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:15,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:15,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:15,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34994978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:15,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:15,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:15,113 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@524718c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,114 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:15,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:15,115 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc8b5e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:15,116 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:15,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@520cedcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,117 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:15,118 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:15,118 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a27423c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:15,120 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cc15946, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,120 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:15,121 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:15,121 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@557467c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:15,122 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:15,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@718cb4d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,124 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:15,124 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:15,124 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:15,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:15,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e833cd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:15,125 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de322ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,127 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:15,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:15,127 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@120081bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:15,129 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35687e6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:15,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:15,131 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7763591f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:15,133 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11330744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:15,135 DEBUG [Thread-1559 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:15,135 DEBUG [Thread-1565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:15,135 DEBUG [Thread-1550 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:15,136 DEBUG [Thread-1563 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:15,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:15,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:15,136 DEBUG [Thread-1546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:15,137 DEBUG [Thread-1548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:15,137 DEBUG [Thread-1561 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:15,138 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:15,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:15,139 DEBUG [Thread-1554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:15,139 DEBUG [Thread-1552 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:15,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:15,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:15,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:15,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:15,141 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:15,141 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:15,141 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:15,141 DEBUG [Thread-1550 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:15,141 DEBUG [Thread-1550 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:15,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:41:15,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:41:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:41:15,143 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:41:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:41:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:41:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:15,145 DEBUG [Thread-1550 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:41:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:41:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:15,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:15,147 DEBUG [Thread-1557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:15,147 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:41:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-11-13T05:41:15,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:15,151 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-13T05:41:15,152 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:15,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:41:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,161 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:15,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:41:15,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a019128cd14d45d7aaa68eaf8ece1d12_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476475147/Put/seqid=0 2024-11-13T05:41:15,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742165_1341 (size=12154) 2024-11-13T05:41:15,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476535205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476535205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476535207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476535208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476535215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-13T05:41:15,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-13T05:41:15,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:15,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:15,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:15,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:15,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:15,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476535327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476535327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476535327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476535327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476535336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-13T05:41:15,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:15,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:15,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:15,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:15,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-13T05:41:15,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476535536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476535536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476535536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476535536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476535546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,601 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:15,605 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a019128cd14d45d7aaa68eaf8ece1d12_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a019128cd14d45d7aaa68eaf8ece1d12_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:15,606 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/da0f12681b024747a2f7f1667612dc08, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:15,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/da0f12681b024747a2f7f1667612dc08 is 175, key is test_row_0/A:col10/1731476475147/Put/seqid=0 2024-11-13T05:41:15,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742166_1342 (size=30955) 2024-11-13T05:41:15,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-13T05:41:15,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:15,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:15,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:15,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:15,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:15,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-13T05:41:15,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:15,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:15,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:15,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:15,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-13T05:41:15,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476535846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476535846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476535846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476535856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476535856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:15,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-13T05:41:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:15,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,013 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/da0f12681b024747a2f7f1667612dc08 2024-11-13T05:41:16,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/40c0b2dd3fb643bc8110a3c9737a15a7 is 50, key is test_row_0/B:col10/1731476475147/Put/seqid=0 2024-11-13T05:41:16,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742167_1343 (size=12001) 2024-11-13T05:41:16,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-13T05:41:16,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:16,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:16,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:16,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-13T05:41:16,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:16,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:16,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:16,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-13T05:41:16,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476536356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:16,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476536356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:16,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:16,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476536356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:16,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:16,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476536366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:16,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476536366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:16,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:16,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:16,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:16,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:16,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:16,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:16,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-13T05:41:16,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:16,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:16,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:16,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/40c0b2dd3fb643bc8110a3c9737a15a7 2024-11-13T05:41:16,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/175b99d715584e9ba4328f41b3f8be4f is 50, key is test_row_0/C:col10/1731476475147/Put/seqid=0 2024-11-13T05:41:16,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742168_1344 (size=12001) 2024-11-13T05:41:16,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-13T05:41:16,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:16,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:16,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:16,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-13T05:41:16,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:16,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:16,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:16,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,738 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:41:16,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-13T05:41:16,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:16,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:16,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:16,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:16,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/175b99d715584e9ba4328f41b3f8be4f 2024-11-13T05:41:16,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/da0f12681b024747a2f7f1667612dc08 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/da0f12681b024747a2f7f1667612dc08 2024-11-13T05:41:16,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/da0f12681b024747a2f7f1667612dc08, entries=150, sequenceid=16, filesize=30.2 K 2024-11-13T05:41:16,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/40c0b2dd3fb643bc8110a3c9737a15a7 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/40c0b2dd3fb643bc8110a3c9737a15a7 2024-11-13T05:41:16,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/40c0b2dd3fb643bc8110a3c9737a15a7, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T05:41:16,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/175b99d715584e9ba4328f41b3f8be4f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/175b99d715584e9ba4328f41b3f8be4f 2024-11-13T05:41:16,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/175b99d715584e9ba4328f41b3f8be4f, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T05:41:16,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for cebd6d1afed0df6f58e14d0610d86767 in 1721ms, sequenceid=16, compaction requested=false 2024-11-13T05:41:16,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:16,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-13T05:41:16,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:16,989 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T05:41:16,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136e20cf262ec74664a7e858bf4f10c0b1_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476475207/Put/seqid=0 2024-11-13T05:41:17,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742169_1345 (size=12154) 2024-11-13T05:41:17,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:17,016 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136e20cf262ec74664a7e858bf4f10c0b1_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136e20cf262ec74664a7e858bf4f10c0b1_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:17,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/51c778673c374336ab956c3be8daebda, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:17,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/51c778673c374336ab956c3be8daebda is 175, key is test_row_0/A:col10/1731476475207/Put/seqid=0 2024-11-13T05:41:17,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742170_1346 (size=30955) 2024-11-13T05:41:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-13T05:41:17,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:17,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476537381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476537382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476537384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476537385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476537387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,436 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/51c778673c374336ab956c3be8daebda 2024-11-13T05:41:17,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/6b8a6c68adae47faa1b97048437f0560 is 50, key is test_row_0/B:col10/1731476475207/Put/seqid=0 2024-11-13T05:41:17,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742171_1347 (size=12001) 2024-11-13T05:41:17,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476537503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476537514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476537514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476537514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476537514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476537716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476537727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476537727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476537727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476537727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:17,847 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/6b8a6c68adae47faa1b97048437f0560 2024-11-13T05:41:17,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/44d42e151c12473c8d845694971fca5a is 50, key is test_row_0/C:col10/1731476475207/Put/seqid=0 2024-11-13T05:41:17,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742172_1348 (size=12001) 2024-11-13T05:41:18,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476538026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476538037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476538037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476538037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476538046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,260 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/44d42e151c12473c8d845694971fca5a 2024-11-13T05:41:18,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/51c778673c374336ab956c3be8daebda as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/51c778673c374336ab956c3be8daebda 2024-11-13T05:41:18,270 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/51c778673c374336ab956c3be8daebda, entries=150, sequenceid=40, filesize=30.2 K 2024-11-13T05:41:18,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/6b8a6c68adae47faa1b97048437f0560 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/6b8a6c68adae47faa1b97048437f0560 2024-11-13T05:41:18,276 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/6b8a6c68adae47faa1b97048437f0560, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T05:41:18,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/44d42e151c12473c8d845694971fca5a as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/44d42e151c12473c8d845694971fca5a 2024-11-13T05:41:18,282 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/44d42e151c12473c8d845694971fca5a, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T05:41:18,283 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for cebd6d1afed0df6f58e14d0610d86767 in 1294ms, sequenceid=40, compaction requested=false 2024-11-13T05:41:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-11-13T05:41:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-11-13T05:41:18,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-13T05:41:18,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1330 sec 2024-11-13T05:41:18,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 3.1440 sec 2024-11-13T05:41:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:18,551 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:18,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T05:41:18,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139f3ec59c893a459ba3476fd96cd7be4a_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476478551/Put/seqid=0 2024-11-13T05:41:18,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742173_1349 (size=17034) 2024-11-13T05:41:18,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476538596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476538596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476538598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476538611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476538612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476538716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476538716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476538716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476538727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476538728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476538936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476538937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476538937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476538937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476538946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:18,971 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:18,981 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139f3ec59c893a459ba3476fd96cd7be4a_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139f3ec59c893a459ba3476fd96cd7be4a_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:18,984 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/1e7a903a77ed43ee9bdfe5a0755f76b3, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:18,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/1e7a903a77ed43ee9bdfe5a0755f76b3 is 175, key is test_row_0/A:col10/1731476478551/Put/seqid=0 2024-11-13T05:41:18,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742174_1350 (size=48139) 2024-11-13T05:41:18,998 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/1e7a903a77ed43ee9bdfe5a0755f76b3 2024-11-13T05:41:19,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/3301d2eddcfc4acdb6e61e2d8223b84c is 50, key is test_row_0/B:col10/1731476478551/Put/seqid=0 2024-11-13T05:41:19,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742175_1351 (size=12001) 2024-11-13T05:41:19,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476539257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:19,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476539257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:19,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:19,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476539257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:19,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:19,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476539257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:19,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476539257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-13T05:41:19,318 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:19,318 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-13T05:41:19,320 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T05:41:19,321 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:19,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T05:41:19,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/3301d2eddcfc4acdb6e61e2d8223b84c 2024-11-13T05:41:19,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/f8df0a2e9d4b4bd5bc1032643d930cea is 50, key is test_row_0/C:col10/1731476478551/Put/seqid=0 2024-11-13T05:41:19,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T05:41:19,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:19,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:19,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:19,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:19,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:19,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742176_1352 (size=12001) 2024-11-13T05:41:19,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T05:41:19,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:19,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:19,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:19,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:19,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T05:41:19,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T05:41:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:19,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:19,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476539777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:19,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476539777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:19,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476539777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:19,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476539777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:19,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476539777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:19,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/f8df0a2e9d4b4bd5bc1032643d930cea 2024-11-13T05:41:19,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/1e7a903a77ed43ee9bdfe5a0755f76b3 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/1e7a903a77ed43ee9bdfe5a0755f76b3 2024-11-13T05:41:19,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/1e7a903a77ed43ee9bdfe5a0755f76b3, entries=250, sequenceid=54, filesize=47.0 K 2024-11-13T05:41:19,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/3301d2eddcfc4acdb6e61e2d8223b84c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/3301d2eddcfc4acdb6e61e2d8223b84c 2024-11-13T05:41:19,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/3301d2eddcfc4acdb6e61e2d8223b84c, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T05:41:19,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/f8df0a2e9d4b4bd5bc1032643d930cea as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/f8df0a2e9d4b4bd5bc1032643d930cea 2024-11-13T05:41:19,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/f8df0a2e9d4b4bd5bc1032643d930cea, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T05:41:19,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for cebd6d1afed0df6f58e14d0610d86767 in 1377ms, sequenceid=54, compaction requested=true 2024-11-13T05:41:19,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:19,929 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:19,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:19,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:19,930 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:19,930 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:19,930 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/A is initiating minor compaction (all files) 2024-11-13T05:41:19,931 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/A in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:19,931 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/da0f12681b024747a2f7f1667612dc08, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/51c778673c374336ab956c3be8daebda, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/1e7a903a77ed43ee9bdfe5a0755f76b3] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=107.5 K 2024-11-13T05:41:19,931 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:19,931 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/da0f12681b024747a2f7f1667612dc08, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/51c778673c374336ab956c3be8daebda, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/1e7a903a77ed43ee9bdfe5a0755f76b3] 2024-11-13T05:41:19,931 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting da0f12681b024747a2f7f1667612dc08, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731476475147 2024-11-13T05:41:19,932 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:19,932 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/B is initiating minor compaction (all files) 2024-11-13T05:41:19,932 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/B in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:19,932 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/40c0b2dd3fb643bc8110a3c9737a15a7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/6b8a6c68adae47faa1b97048437f0560, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/3301d2eddcfc4acdb6e61e2d8223b84c] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=35.2 K 2024-11-13T05:41:19,932 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51c778673c374336ab956c3be8daebda, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731476475205 2024-11-13T05:41:19,932 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 40c0b2dd3fb643bc8110a3c9737a15a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731476475147 2024-11-13T05:41:19,932 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e7a903a77ed43ee9bdfe5a0755f76b3, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731476477376 2024-11-13T05:41:19,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:19,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:19,933 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b8a6c68adae47faa1b97048437f0560, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731476475205 2024-11-13T05:41:19,935 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 3301d2eddcfc4acdb6e61e2d8223b84c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731476477376 2024-11-13T05:41:19,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-13T05:41:19,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:19,940 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T05:41:19,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:19,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:19,943 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T05:41:19,965 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#B#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:19,965 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/c0551510c20849719111acbbb85992c9 is 50, key is test_row_0/B:col10/1731476478551/Put/seqid=0 2024-11-13T05:41:19,971 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113c92bc11fe5fb4c708bf0442472c3ae74_cebd6d1afed0df6f58e14d0610d86767 store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:19,973 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113c92bc11fe5fb4c708bf0442472c3ae74_cebd6d1afed0df6f58e14d0610d86767, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:19,974 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c92bc11fe5fb4c708bf0442472c3ae74_cebd6d1afed0df6f58e14d0610d86767 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:19,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bea6bd7f5759469a904c74a5dd8d340c_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476478610/Put/seqid=0 2024-11-13T05:41:20,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742177_1353 (size=12104) 2024-11-13T05:41:20,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742178_1354 (size=4469) 2024-11-13T05:41:20,032 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#A#compaction#300 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:20,033 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/dd69c76b2e7141a1b784edf1832090a1 is 175, key is test_row_0/A:col10/1731476478551/Put/seqid=0 2024-11-13T05:41:20,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742179_1355 (size=12154) 2024-11-13T05:41:20,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742180_1356 (size=31058) 2024-11-13T05:41:20,065 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/dd69c76b2e7141a1b784edf1832090a1 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/dd69c76b2e7141a1b784edf1832090a1 2024-11-13T05:41:20,083 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/A of cebd6d1afed0df6f58e14d0610d86767 into dd69c76b2e7141a1b784edf1832090a1(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:20,083 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:20,083 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/A, priority=13, startTime=1731476479929; duration=0sec 2024-11-13T05:41:20,083 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:20,083 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:A 2024-11-13T05:41:20,083 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:20,086 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:20,086 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/C is initiating minor compaction (all files) 2024-11-13T05:41:20,086 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/C in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:20,087 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/175b99d715584e9ba4328f41b3f8be4f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/44d42e151c12473c8d845694971fca5a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/f8df0a2e9d4b4bd5bc1032643d930cea] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=35.2 K 2024-11-13T05:41:20,087 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 175b99d715584e9ba4328f41b3f8be4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731476475147 2024-11-13T05:41:20,087 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44d42e151c12473c8d845694971fca5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731476475205 2024-11-13T05:41:20,088 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8df0a2e9d4b4bd5bc1032643d930cea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731476477376 2024-11-13T05:41:20,104 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#C#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:20,104 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/aed3607597934e69a140486011ce193b is 50, key is test_row_0/C:col10/1731476478551/Put/seqid=0 2024-11-13T05:41:20,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742181_1357 (size=12104) 2024-11-13T05:41:20,159 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/aed3607597934e69a140486011ce193b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/aed3607597934e69a140486011ce193b 2024-11-13T05:41:20,170 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/C of cebd6d1afed0df6f58e14d0610d86767 into aed3607597934e69a140486011ce193b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:20,170 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:20,170 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/C, priority=13, startTime=1731476479939; duration=0sec 2024-11-13T05:41:20,170 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:20,170 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:C 2024-11-13T05:41:20,418 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/c0551510c20849719111acbbb85992c9 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c0551510c20849719111acbbb85992c9 2024-11-13T05:41:20,423 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/B of cebd6d1afed0df6f58e14d0610d86767 into c0551510c20849719111acbbb85992c9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:20,424 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:20,424 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/B, priority=13, startTime=1731476479930; duration=0sec 2024-11-13T05:41:20,424 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:20,424 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:B 2024-11-13T05:41:20,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:20,445 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bea6bd7f5759469a904c74a5dd8d340c_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bea6bd7f5759469a904c74a5dd8d340c_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:20,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/97ec014b77f44c2ea3fc594ed211f390, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:20,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/97ec014b77f44c2ea3fc594ed211f390 is 175, key is test_row_0/A:col10/1731476478610/Put/seqid=0 2024-11-13T05:41:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T05:41:20,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742182_1358 (size=30955) 2024-11-13T05:41:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:20,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:20,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476540815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:20,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476540817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:20,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476540818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:20,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:20,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476540823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476540824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:20,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:20,867 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/97ec014b77f44c2ea3fc594ed211f390 2024-11-13T05:41:20,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/d72a7d1027fb4af399312236d3247fa5 is 50, key is test_row_0/B:col10/1731476478610/Put/seqid=0 2024-11-13T05:41:20,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742183_1359 (size=12001) 2024-11-13T05:41:20,904 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/d72a7d1027fb4af399312236d3247fa5 2024-11-13T05:41:20,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/40b3bfbf072a4a9e8c19f5d6a21da298 is 50, key is test_row_0/C:col10/1731476478610/Put/seqid=0 2024-11-13T05:41:20,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476540936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:20,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:20,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476540937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:20,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:20,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476540937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:20,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:20,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476540937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:20,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:20,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476540937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:20,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:20,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:20,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742184_1360 (size=12001) 2024-11-13T05:41:21,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476541146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476541146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476541157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476541157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476541157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,383 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/40b3bfbf072a4a9e8c19f5d6a21da298 2024-11-13T05:41:21,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/97ec014b77f44c2ea3fc594ed211f390 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/97ec014b77f44c2ea3fc594ed211f390 2024-11-13T05:41:21,390 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/97ec014b77f44c2ea3fc594ed211f390, entries=150, sequenceid=76, filesize=30.2 K 2024-11-13T05:41:21,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/d72a7d1027fb4af399312236d3247fa5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/d72a7d1027fb4af399312236d3247fa5 2024-11-13T05:41:21,395 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/d72a7d1027fb4af399312236d3247fa5, entries=150, sequenceid=76, filesize=11.7 K 2024-11-13T05:41:21,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/40b3bfbf072a4a9e8c19f5d6a21da298 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/40b3bfbf072a4a9e8c19f5d6a21da298 2024-11-13T05:41:21,400 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/40b3bfbf072a4a9e8c19f5d6a21da298, entries=150, sequenceid=76, filesize=11.7 K 2024-11-13T05:41:21,401 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for cebd6d1afed0df6f58e14d0610d86767 in 1462ms, sequenceid=76, compaction requested=false 2024-11-13T05:41:21,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:21,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:21,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-13T05:41:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-13T05:41:21,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-13T05:41:21,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0820 sec 2024-11-13T05:41:21,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 2.0870 sec 2024-11-13T05:41:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:21,472 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:21,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-13T05:41:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T05:41:21,477 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:21,478 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-13T05:41:21,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131e6a457b002649ceb9d9a98d4a285a52_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476481471/Put/seqid=0 2024-11-13T05:41:21,480 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T05:41:21,482 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:21,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:21,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742185_1361 (size=14594) 2024-11-13T05:41:21,497 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:21,502 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131e6a457b002649ceb9d9a98d4a285a52_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131e6a457b002649ceb9d9a98d4a285a52_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:21,503 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b93f9774f6f048e997f46aa682798d78, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:21,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b93f9774f6f048e997f46aa682798d78 is 175, key is test_row_0/A:col10/1731476481471/Put/seqid=0 2024-11-13T05:41:21,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742186_1362 (size=39549) 2024-11-13T05:41:21,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476541517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476541518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476541519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476541520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476541521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T05:41:21,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T05:41:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:21,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:21,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:21,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:21,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:21,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476541636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476541647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476541647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476541647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476541647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T05:41:21,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:21,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T05:41:21,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476541847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476541871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476541871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476541871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476541871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:21,912 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b93f9774f6f048e997f46aa682798d78 2024-11-13T05:41:21,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/25f2e98ef26e483aab25f6be07d23f74 is 50, key is test_row_0/B:col10/1731476481471/Put/seqid=0 2024-11-13T05:41:21,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742187_1363 (size=12001) 2024-11-13T05:41:21,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T05:41:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:21,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:21,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T05:41:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T05:41:22,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476542156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:22,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476542186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476542186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476542186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476542186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:22,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T05:41:22,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:22,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/25f2e98ef26e483aab25f6be07d23f74 2024-11-13T05:41:22,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/3b5f913a93c24668a4de670935457bd7 is 50, key is test_row_0/C:col10/1731476481471/Put/seqid=0 2024-11-13T05:41:22,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742188_1364 (size=12001) 2024-11-13T05:41:22,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T05:41:22,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:22,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T05:41:22,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:22,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T05:41:22,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476542676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:22,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476542697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:22,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476542697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:22,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T05:41:22,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:22,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:22,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476542706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:22,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476542706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:22,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/3b5f913a93c24668a4de670935457bd7 2024-11-13T05:41:22,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b93f9774f6f048e997f46aa682798d78 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b93f9774f6f048e997f46aa682798d78 2024-11-13T05:41:22,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b93f9774f6f048e997f46aa682798d78, entries=200, sequenceid=94, filesize=38.6 K 2024-11-13T05:41:22,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/25f2e98ef26e483aab25f6be07d23f74 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/25f2e98ef26e483aab25f6be07d23f74 2024-11-13T05:41:22,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/25f2e98ef26e483aab25f6be07d23f74, entries=150, sequenceid=94, filesize=11.7 K 2024-11-13T05:41:22,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/3b5f913a93c24668a4de670935457bd7 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/3b5f913a93c24668a4de670935457bd7 2024-11-13T05:41:22,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/3b5f913a93c24668a4de670935457bd7, entries=150, sequenceid=94, filesize=11.7 K 2024-11-13T05:41:22,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for cebd6d1afed0df6f58e14d0610d86767 in 1311ms, sequenceid=94, compaction requested=true 2024-11-13T05:41:22,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:22,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:22,784 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:22,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:22,784 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:22,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:22,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:22,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:22,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:22,788 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:22,788 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:22,788 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/A is initiating minor compaction (all files) 2024-11-13T05:41:22,788 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/B is initiating minor compaction (all files) 2024-11-13T05:41:22,788 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/A in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,788 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/B in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,788 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/dd69c76b2e7141a1b784edf1832090a1, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/97ec014b77f44c2ea3fc594ed211f390, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b93f9774f6f048e997f46aa682798d78] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=99.2 K 2024-11-13T05:41:22,788 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c0551510c20849719111acbbb85992c9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/d72a7d1027fb4af399312236d3247fa5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/25f2e98ef26e483aab25f6be07d23f74] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=35.3 K 2024-11-13T05:41:22,788 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,788 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/dd69c76b2e7141a1b784edf1832090a1, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/97ec014b77f44c2ea3fc594ed211f390, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b93f9774f6f048e997f46aa682798d78] 2024-11-13T05:41:22,789 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting c0551510c20849719111acbbb85992c9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731476477376 2024-11-13T05:41:22,789 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd69c76b2e7141a1b784edf1832090a1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731476477376 2024-11-13T05:41:22,789 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting d72a7d1027fb4af399312236d3247fa5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731476478596 2024-11-13T05:41:22,789 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 25f2e98ef26e483aab25f6be07d23f74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731476480821 2024-11-13T05:41:22,789 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97ec014b77f44c2ea3fc594ed211f390, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731476478596 2024-11-13T05:41:22,790 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting b93f9774f6f048e997f46aa682798d78, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731476480816 2024-11-13T05:41:22,798 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:22,798 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/0288c04bb54443d6b511829c7a1e983a is 50, key is test_row_0/B:col10/1731476481471/Put/seqid=0 2024-11-13T05:41:22,799 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:22,802 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411135e48d3342485428c995530fe4512dfa1_cebd6d1afed0df6f58e14d0610d86767 store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:22,805 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411135e48d3342485428c995530fe4512dfa1_cebd6d1afed0df6f58e14d0610d86767, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:22,805 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135e48d3342485428c995530fe4512dfa1_cebd6d1afed0df6f58e14d0610d86767 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:22,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742189_1365 (size=12207) 2024-11-13T05:41:22,815 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/0288c04bb54443d6b511829c7a1e983a as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/0288c04bb54443d6b511829c7a1e983a 2024-11-13T05:41:22,820 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/B of cebd6d1afed0df6f58e14d0610d86767 into 0288c04bb54443d6b511829c7a1e983a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:22,820 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:22,821 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/B, priority=13, startTime=1731476482784; duration=0sec 2024-11-13T05:41:22,821 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:22,824 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:B 2024-11-13T05:41:22,824 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:22,825 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:22,825 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/C is initiating minor compaction (all files) 2024-11-13T05:41:22,826 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/C in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,826 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/aed3607597934e69a140486011ce193b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/40b3bfbf072a4a9e8c19f5d6a21da298, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/3b5f913a93c24668a4de670935457bd7] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=35.3 K 2024-11-13T05:41:22,826 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting aed3607597934e69a140486011ce193b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731476477376 2024-11-13T05:41:22,826 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b3bfbf072a4a9e8c19f5d6a21da298, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731476478596 2024-11-13T05:41:22,827 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b5f913a93c24668a4de670935457bd7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731476480821 2024-11-13T05:41:22,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742190_1366 (size=4469) 2024-11-13T05:41:22,843 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:22,844 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/8371a3ae534140dda27079b7d6191577 is 50, key is test_row_0/C:col10/1731476481471/Put/seqid=0 2024-11-13T05:41:22,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742191_1367 (size=12207) 2024-11-13T05:41:22,860 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/8371a3ae534140dda27079b7d6191577 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/8371a3ae534140dda27079b7d6191577 2024-11-13T05:41:22,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-13T05:41:22,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:22,862 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T05:41:22,872 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/C of cebd6d1afed0df6f58e14d0610d86767 into 8371a3ae534140dda27079b7d6191577(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:22,872 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:22,872 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/C, priority=13, startTime=1731476482784; duration=0sec 2024-11-13T05:41:22,872 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:22,872 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:C 2024-11-13T05:41:22,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ca0adf5b778b4b8787dda616e5a0afad_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476481519/Put/seqid=0 2024-11-13T05:41:22,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742192_1368 (size=12154) 2024-11-13T05:41:23,247 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#A#compaction#310 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:23,247 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/49d0bcb399f44b6389bebb920855e9c4 is 175, key is test_row_0/A:col10/1731476481471/Put/seqid=0 2024-11-13T05:41:23,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742193_1369 (size=31161) 2024-11-13T05:41:23,257 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/49d0bcb399f44b6389bebb920855e9c4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/49d0bcb399f44b6389bebb920855e9c4 2024-11-13T05:41:23,261 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/A of cebd6d1afed0df6f58e14d0610d86767 into 49d0bcb399f44b6389bebb920855e9c4(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:23,261 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:23,261 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/A, priority=13, startTime=1731476482784; duration=0sec 2024-11-13T05:41:23,262 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:23,262 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:A 2024-11-13T05:41:23,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:23,291 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ca0adf5b778b4b8787dda616e5a0afad_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ca0adf5b778b4b8787dda616e5a0afad_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:23,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/0ea64484836545f1b5ebedd99ae8f56f, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:23,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/0ea64484836545f1b5ebedd99ae8f56f is 175, key is test_row_0/A:col10/1731476481519/Put/seqid=0 2024-11-13T05:41:23,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742194_1370 (size=30955) 2024-11-13T05:41:23,305 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/0ea64484836545f1b5ebedd99ae8f56f 2024-11-13T05:41:23,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/00a4cf7ee70b4d3586e843f5af3f50b2 is 50, key is test_row_0/B:col10/1731476481519/Put/seqid=0 2024-11-13T05:41:23,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742195_1371 (size=12001) 2024-11-13T05:41:23,341 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/00a4cf7ee70b4d3586e843f5af3f50b2 2024-11-13T05:41:23,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/05343cd96fd94b32b6d4d6a087f718cc is 50, key is test_row_0/C:col10/1731476481519/Put/seqid=0 2024-11-13T05:41:23,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742196_1372 (size=12001) 2024-11-13T05:41:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T05:41:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:23,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:23,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:23,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476543726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476543723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:23,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476543726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:23,736 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in cebd6d1afed0df6f58e14d0610d86767 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:23,737 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_0' in cebd6d1afed0df6f58e14d0610d86767 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:23,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476543736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:23,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:23,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476543738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:23,781 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/05343cd96fd94b32b6d4d6a087f718cc 2024-11-13T05:41:23,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/0ea64484836545f1b5ebedd99ae8f56f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0ea64484836545f1b5ebedd99ae8f56f 2024-11-13T05:41:23,790 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0ea64484836545f1b5ebedd99ae8f56f, entries=150, sequenceid=116, filesize=30.2 K 2024-11-13T05:41:23,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/00a4cf7ee70b4d3586e843f5af3f50b2 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/00a4cf7ee70b4d3586e843f5af3f50b2 2024-11-13T05:41:23,796 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/00a4cf7ee70b4d3586e843f5af3f50b2, entries=150, sequenceid=116, filesize=11.7 K 2024-11-13T05:41:23,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/05343cd96fd94b32b6d4d6a087f718cc as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/05343cd96fd94b32b6d4d6a087f718cc 2024-11-13T05:41:23,803 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/05343cd96fd94b32b6d4d6a087f718cc, entries=150, sequenceid=116, filesize=11.7 K 2024-11-13T05:41:23,804 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for cebd6d1afed0df6f58e14d0610d86767 in 942ms, sequenceid=116, compaction requested=false 2024-11-13T05:41:23,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:23,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:23,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-13T05:41:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-13T05:41:23,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-13T05:41:23,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3240 sec 2024-11-13T05:41:23,811 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.3310 sec 2024-11-13T05:41:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:23,851 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:23,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T05:41:23,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135a29b2d9f7a04b019e27a10f3dcf6994_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476483850/Put/seqid=0 2024-11-13T05:41:23,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742197_1373 (size=14794) 2024-11-13T05:41:23,891 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:23,897 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135a29b2d9f7a04b019e27a10f3dcf6994_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135a29b2d9f7a04b019e27a10f3dcf6994_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:23,900 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/1b94be91aef94a609faa94ec1282376f, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:23,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/1b94be91aef94a609faa94ec1282376f is 175, key is test_row_0/A:col10/1731476483850/Put/seqid=0 2024-11-13T05:41:23,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742198_1374 (size=39749) 2024-11-13T05:41:23,907 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/1b94be91aef94a609faa94ec1282376f 2024-11-13T05:41:23,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/fe679ebfa1794f0c85eacbb52e56c366 is 50, key is test_row_0/B:col10/1731476483850/Put/seqid=0 2024-11-13T05:41:23,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742199_1375 (size=12151) 2024-11-13T05:41:23,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/fe679ebfa1794f0c85eacbb52e56c366 2024-11-13T05:41:23,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/4d928a00f6dd4b589b9d15a90c37d75c is 50, key is test_row_0/C:col10/1731476483850/Put/seqid=0 2024-11-13T05:41:23,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476543946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476543947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:23,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476543948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:23,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742200_1376 (size=12151) 2024-11-13T05:41:23,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/4d928a00f6dd4b589b9d15a90c37d75c 2024-11-13T05:41:23,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/1b94be91aef94a609faa94ec1282376f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/1b94be91aef94a609faa94ec1282376f 2024-11-13T05:41:23,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/1b94be91aef94a609faa94ec1282376f, entries=200, sequenceid=135, filesize=38.8 K 2024-11-13T05:41:23,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/fe679ebfa1794f0c85eacbb52e56c366 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/fe679ebfa1794f0c85eacbb52e56c366 2024-11-13T05:41:23,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/fe679ebfa1794f0c85eacbb52e56c366, entries=150, sequenceid=135, filesize=11.9 K 2024-11-13T05:41:23,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/4d928a00f6dd4b589b9d15a90c37d75c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/4d928a00f6dd4b589b9d15a90c37d75c 2024-11-13T05:41:23,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/4d928a00f6dd4b589b9d15a90c37d75c, entries=150, sequenceid=135, filesize=11.9 K 2024-11-13T05:41:24,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for cebd6d1afed0df6f58e14d0610d86767 in 148ms, sequenceid=135, compaction requested=true 2024-11-13T05:41:24,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:24,000 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:24,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:24,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:24,000 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:24,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:24,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:24,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:24,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:24,001 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:24,001 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/A is initiating minor compaction (all files) 2024-11-13T05:41:24,001 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/A in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:24,001 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/49d0bcb399f44b6389bebb920855e9c4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0ea64484836545f1b5ebedd99ae8f56f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/1b94be91aef94a609faa94ec1282376f] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=99.5 K 2024-11-13T05:41:24,001 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:24,001 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/49d0bcb399f44b6389bebb920855e9c4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0ea64484836545f1b5ebedd99ae8f56f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/1b94be91aef94a609faa94ec1282376f] 2024-11-13T05:41:24,002 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49d0bcb399f44b6389bebb920855e9c4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731476480821 2024-11-13T05:41:24,002 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:24,002 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/B is initiating minor compaction (all files) 2024-11-13T05:41:24,002 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/B in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:24,002 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/0288c04bb54443d6b511829c7a1e983a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/00a4cf7ee70b4d3586e843f5af3f50b2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/fe679ebfa1794f0c85eacbb52e56c366] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=35.5 K 2024-11-13T05:41:24,003 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 0288c04bb54443d6b511829c7a1e983a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731476480821 2024-11-13T05:41:24,003 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ea64484836545f1b5ebedd99ae8f56f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731476481517 2024-11-13T05:41:24,003 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 00a4cf7ee70b4d3586e843f5af3f50b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731476481517 2024-11-13T05:41:24,003 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b94be91aef94a609faa94ec1282376f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731476483720 2024-11-13T05:41:24,003 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting fe679ebfa1794f0c85eacbb52e56c366, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731476483720 2024-11-13T05:41:24,021 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#B#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:24,021 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/a709864c2e3f40a9a3128db5c0706395 is 50, key is test_row_0/B:col10/1731476483850/Put/seqid=0 2024-11-13T05:41:24,030 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:24,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742201_1377 (size=12459) 2024-11-13T05:41:24,051 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111337c68fd08d1d4770833a85a8324f8756_cebd6d1afed0df6f58e14d0610d86767 store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:24,053 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111337c68fd08d1d4770833a85a8324f8756_cebd6d1afed0df6f58e14d0610d86767, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:24,054 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111337c68fd08d1d4770833a85a8324f8756_cebd6d1afed0df6f58e14d0610d86767 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:24,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742202_1378 (size=4469) 2024-11-13T05:41:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:24,069 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:24,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T05:41:24,069 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#A#compaction#319 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:24,070 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/cc37ca1a31a64e1eb7997fe3ab90ca2a is 175, key is test_row_0/A:col10/1731476483850/Put/seqid=0 2024-11-13T05:41:24,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742203_1379 (size=31413) 2024-11-13T05:41:24,110 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/cc37ca1a31a64e1eb7997fe3ab90ca2a as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/cc37ca1a31a64e1eb7997fe3ab90ca2a 2024-11-13T05:41:24,115 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/A of cebd6d1afed0df6f58e14d0610d86767 into cc37ca1a31a64e1eb7997fe3ab90ca2a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:24,115 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:24,115 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/A, priority=13, startTime=1731476484000; duration=0sec 2024-11-13T05:41:24,115 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:24,115 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:A 2024-11-13T05:41:24,116 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:24,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111358a12772241d4664b939d167524988a6_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476483942/Put/seqid=0 2024-11-13T05:41:24,122 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:24,122 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/C is initiating minor compaction (all files) 2024-11-13T05:41:24,122 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/C in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:24,122 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/8371a3ae534140dda27079b7d6191577, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/05343cd96fd94b32b6d4d6a087f718cc, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/4d928a00f6dd4b589b9d15a90c37d75c] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=35.5 K 2024-11-13T05:41:24,123 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8371a3ae534140dda27079b7d6191577, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731476480821 2024-11-13T05:41:24,123 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05343cd96fd94b32b6d4d6a087f718cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731476481517 2024-11-13T05:41:24,124 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d928a00f6dd4b589b9d15a90c37d75c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731476483720 2024-11-13T05:41:24,145 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#C#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:24,145 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/6c49085414b34ad0aed809d2a546b125 is 50, key is test_row_0/C:col10/1731476483850/Put/seqid=0 2024-11-13T05:41:24,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476544134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:24,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:24,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476544134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:24,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:24,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:24,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476544136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:24,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742204_1380 (size=17284) 2024-11-13T05:41:24,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742205_1381 (size=12459) 2024-11-13T05:41:24,185 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/6c49085414b34ad0aed809d2a546b125 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/6c49085414b34ad0aed809d2a546b125 2024-11-13T05:41:24,194 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/C of cebd6d1afed0df6f58e14d0610d86767 into 6c49085414b34ad0aed809d2a546b125(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:24,194 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:24,194 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/C, priority=13, startTime=1731476484000; duration=0sec 2024-11-13T05:41:24,194 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:24,194 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:C 2024-11-13T05:41:24,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476544256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:24,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476544256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:24,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476544256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:24,448 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/a709864c2e3f40a9a3128db5c0706395 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/a709864c2e3f40a9a3128db5c0706395 2024-11-13T05:41:24,452 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/B of cebd6d1afed0df6f58e14d0610d86767 into a709864c2e3f40a9a3128db5c0706395(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:24,452 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:24,452 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/B, priority=13, startTime=1731476484000; duration=0sec 2024-11-13T05:41:24,452 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:24,452 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:B 2024-11-13T05:41:24,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476544467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:24,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476544467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:24,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476544467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:24,555 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:24,558 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111358a12772241d4664b939d167524988a6_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111358a12772241d4664b939d167524988a6_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:24,559 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/072ddde985984d818e8a54d1dd91324a, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:24,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/072ddde985984d818e8a54d1dd91324a is 175, key is test_row_0/A:col10/1731476483942/Put/seqid=0 2024-11-13T05:41:24,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742206_1382 (size=48389) 2024-11-13T05:41:24,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476544776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:24,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476544776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:24,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476544776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:24,964 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/072ddde985984d818e8a54d1dd91324a 2024-11-13T05:41:24,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/c676e48441d740b2812e0f09813b537e is 50, key is test_row_0/B:col10/1731476483942/Put/seqid=0 2024-11-13T05:41:24,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742207_1383 (size=12151) 2024-11-13T05:41:24,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/c676e48441d740b2812e0f09813b537e 2024-11-13T05:41:25,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/390a15d25cd3470b8f1ac44fdb510f2f is 50, key is test_row_0/C:col10/1731476483942/Put/seqid=0 2024-11-13T05:41:25,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742208_1384 (size=12151) 2024-11-13T05:41:25,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/390a15d25cd3470b8f1ac44fdb510f2f 2024-11-13T05:41:25,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/072ddde985984d818e8a54d1dd91324a as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/072ddde985984d818e8a54d1dd91324a 2024-11-13T05:41:25,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/072ddde985984d818e8a54d1dd91324a, entries=250, sequenceid=156, filesize=47.3 K 2024-11-13T05:41:25,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/c676e48441d740b2812e0f09813b537e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c676e48441d740b2812e0f09813b537e 2024-11-13T05:41:25,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c676e48441d740b2812e0f09813b537e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-13T05:41:25,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/390a15d25cd3470b8f1ac44fdb510f2f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/390a15d25cd3470b8f1ac44fdb510f2f 2024-11-13T05:41:25,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/390a15d25cd3470b8f1ac44fdb510f2f, entries=150, sequenceid=156, filesize=11.9 K 2024-11-13T05:41:25,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for cebd6d1afed0df6f58e14d0610d86767 in 983ms, sequenceid=156, compaction requested=false 2024-11-13T05:41:25,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:25,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:25,301 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:25,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T05:41:25,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135b935f4971744200bbd3aafde5a1efad_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476484132/Put/seqid=0 2024-11-13T05:41:25,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742209_1385 (size=14794) 2024-11-13T05:41:25,311 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:25,314 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135b935f4971744200bbd3aafde5a1efad_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135b935f4971744200bbd3aafde5a1efad_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:25,315 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b2cccc31abc14edd9f3bdff86a7fc6fa, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:25,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b2cccc31abc14edd9f3bdff86a7fc6fa is 175, key is test_row_0/A:col10/1731476484132/Put/seqid=0 2024-11-13T05:41:25,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742210_1386 (size=39749) 2024-11-13T05:41:25,319 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b2cccc31abc14edd9f3bdff86a7fc6fa 2024-11-13T05:41:25,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/0b52b8dc89d742e19ede3e70a3323473 is 50, key is test_row_0/B:col10/1731476484132/Put/seqid=0 2024-11-13T05:41:25,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742211_1387 (size=12151) 2024-11-13T05:41:25,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:25,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476545356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:25,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476545360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476545360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:25,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476545466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476545466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:25,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476545476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:25,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-13T05:41:25,636 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:25,636 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-13T05:41:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T05:41:25,638 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:25,639 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:25,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:25,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476545676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476545676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:25,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476545687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:25,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/0b52b8dc89d742e19ede3e70a3323473 2024-11-13T05:41:25,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/4beb4a2a5dd54e80b86ff6de97de8a21 is 50, key is test_row_0/C:col10/1731476484132/Put/seqid=0 2024-11-13T05:41:25,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742212_1388 (size=12151) 2024-11-13T05:41:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T05:41:25,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476545746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:25,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476545746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:25,756 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in cebd6d1afed0df6f58e14d0610d86767 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:25,756 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_0' in cebd6d1afed0df6f58e14d0610d86767 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:25,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T05:41:25,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:25,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:25,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:25,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:25,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:25,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T05:41:25,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:25,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:25,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:25,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:25,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:25,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T05:41:26,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476545996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:26,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476545996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:26,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476546006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:26,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:26,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T05:41:26,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:26,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:26,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:26,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:26,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:26,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/4beb4a2a5dd54e80b86ff6de97de8a21 2024-11-13T05:41:26,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b2cccc31abc14edd9f3bdff86a7fc6fa as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b2cccc31abc14edd9f3bdff86a7fc6fa 2024-11-13T05:41:26,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b2cccc31abc14edd9f3bdff86a7fc6fa, entries=200, sequenceid=175, filesize=38.8 K 2024-11-13T05:41:26,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/0b52b8dc89d742e19ede3e70a3323473 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/0b52b8dc89d742e19ede3e70a3323473 2024-11-13T05:41:26,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/0b52b8dc89d742e19ede3e70a3323473, entries=150, sequenceid=175, filesize=11.9 K 2024-11-13T05:41:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/4beb4a2a5dd54e80b86ff6de97de8a21 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/4beb4a2a5dd54e80b86ff6de97de8a21 2024-11-13T05:41:26,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/4beb4a2a5dd54e80b86ff6de97de8a21, entries=150, sequenceid=175, filesize=11.9 K 2024-11-13T05:41:26,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for cebd6d1afed0df6f58e14d0610d86767 in 864ms, sequenceid=175, compaction requested=true 2024-11-13T05:41:26,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:26,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:26,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:26,165 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:26,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:26,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:26,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:26,165 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:26,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:26,166 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:26,166 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/A is initiating minor compaction (all files) 2024-11-13T05:41:26,167 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/A in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:26,167 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/cc37ca1a31a64e1eb7997fe3ab90ca2a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/072ddde985984d818e8a54d1dd91324a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b2cccc31abc14edd9f3bdff86a7fc6fa] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=116.7 K 2024-11-13T05:41:26,167 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:26,167 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/cc37ca1a31a64e1eb7997fe3ab90ca2a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/072ddde985984d818e8a54d1dd91324a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b2cccc31abc14edd9f3bdff86a7fc6fa] 2024-11-13T05:41:26,167 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc37ca1a31a64e1eb7997fe3ab90ca2a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731476483720 2024-11-13T05:41:26,168 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 072ddde985984d818e8a54d1dd91324a, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731476483925 2024-11-13T05:41:26,169 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:26,169 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/B is initiating minor compaction (all files) 2024-11-13T05:41:26,169 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/B in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:26,169 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/a709864c2e3f40a9a3128db5c0706395, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c676e48441d740b2812e0f09813b537e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/0b52b8dc89d742e19ede3e70a3323473] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=35.9 K 2024-11-13T05:41:26,169 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2cccc31abc14edd9f3bdff86a7fc6fa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731476484086 2024-11-13T05:41:26,169 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting a709864c2e3f40a9a3128db5c0706395, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731476483720 2024-11-13T05:41:26,170 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting c676e48441d740b2812e0f09813b537e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731476483942 2024-11-13T05:41:26,170 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b52b8dc89d742e19ede3e70a3323473, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731476484132 2024-11-13T05:41:26,180 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:26,195 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#B#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:26,196 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/e2dc68cf8a994e27ba5a0fbc4a002abf is 50, key is test_row_0/B:col10/1731476484132/Put/seqid=0 2024-11-13T05:41:26,204 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113ff090232371342fea6a54bdb04c8de1e_cebd6d1afed0df6f58e14d0610d86767 store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:26,206 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113ff090232371342fea6a54bdb04c8de1e_cebd6d1afed0df6f58e14d0610d86767, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:26,207 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ff090232371342fea6a54bdb04c8de1e_cebd6d1afed0df6f58e14d0610d86767 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:26,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-13T05:41:26,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:26,249 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T05:41:26,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742213_1389 (size=12561) 2024-11-13T05:41:26,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742214_1390 (size=4469) 2024-11-13T05:41:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T05:41:26,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139fdf03ce5baf496f8f95ebbc2dee178d_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476485355/Put/seqid=0 2024-11-13T05:41:26,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742215_1391 (size=12304) 2024-11-13T05:41:26,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:26,291 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139fdf03ce5baf496f8f95ebbc2dee178d_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139fdf03ce5baf496f8f95ebbc2dee178d_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:26,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b52bcaeab95e4d498056e9323d00b582, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:26,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b52bcaeab95e4d498056e9323d00b582 is 175, key is test_row_0/A:col10/1731476485355/Put/seqid=0 2024-11-13T05:41:26,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742216_1392 (size=31105) 2024-11-13T05:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:26,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:26,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476546549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:26,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476546549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:26,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476546560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:26,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476546666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:26,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476546666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:26,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:26,678 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#A#compaction#327 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:26,679 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/846b794be6414835a74422a25aaafe08 is 175, key is test_row_0/A:col10/1731476484132/Put/seqid=0 2024-11-13T05:41:26,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:26,683 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/e2dc68cf8a994e27ba5a0fbc4a002abf as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/e2dc68cf8a994e27ba5a0fbc4a002abf 2024-11-13T05:41:26,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476546680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:26,691 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/B of cebd6d1afed0df6f58e14d0610d86767 into e2dc68cf8a994e27ba5a0fbc4a002abf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:26,691 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:26,691 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/B, priority=13, startTime=1731476486165; duration=0sec 2024-11-13T05:41:26,692 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:26,692 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:B 2024-11-13T05:41:26,692 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:26,693 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:26,693 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/C is initiating minor compaction (all files) 2024-11-13T05:41:26,693 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/C in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:26,693 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/6c49085414b34ad0aed809d2a546b125, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/390a15d25cd3470b8f1ac44fdb510f2f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/4beb4a2a5dd54e80b86ff6de97de8a21] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=35.9 K 2024-11-13T05:41:26,693 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c49085414b34ad0aed809d2a546b125, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731476483720 2024-11-13T05:41:26,694 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 390a15d25cd3470b8f1ac44fdb510f2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731476483942 2024-11-13T05:41:26,694 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 4beb4a2a5dd54e80b86ff6de97de8a21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731476484132 2024-11-13T05:41:26,714 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b52bcaeab95e4d498056e9323d00b582 2024-11-13T05:41:26,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742217_1393 (size=31515) 2024-11-13T05:41:26,725 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#C#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:26,725 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/77e1c83912fd45fda2ddbcfec7b80c40 is 50, key is test_row_0/C:col10/1731476484132/Put/seqid=0 2024-11-13T05:41:26,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/00ca894ee5c143ffbfc46130d1d677c5 is 50, key is test_row_0/B:col10/1731476485355/Put/seqid=0 2024-11-13T05:41:26,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742218_1394 (size=12561) 2024-11-13T05:41:26,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742219_1395 (size=12151) 2024-11-13T05:41:26,744 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/00ca894ee5c143ffbfc46130d1d677c5 2024-11-13T05:41:26,747 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/77e1c83912fd45fda2ddbcfec7b80c40 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/77e1c83912fd45fda2ddbcfec7b80c40 2024-11-13T05:41:26,756 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/C of cebd6d1afed0df6f58e14d0610d86767 into 77e1c83912fd45fda2ddbcfec7b80c40(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:26,756 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:26,756 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/C, priority=13, startTime=1731476486165; duration=0sec 2024-11-13T05:41:26,756 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:26,756 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:C 2024-11-13T05:41:26,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/b8eec58646ba4f009eb4fc065580d4ea is 50, key is test_row_0/C:col10/1731476485355/Put/seqid=0 2024-11-13T05:41:26,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742220_1396 (size=12151) 2024-11-13T05:41:26,762 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/b8eec58646ba4f009eb4fc065580d4ea 2024-11-13T05:41:26,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b52bcaeab95e4d498056e9323d00b582 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b52bcaeab95e4d498056e9323d00b582 2024-11-13T05:41:26,775 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b52bcaeab95e4d498056e9323d00b582, entries=150, sequenceid=196, filesize=30.4 K 2024-11-13T05:41:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T05:41:26,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/00ca894ee5c143ffbfc46130d1d677c5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/00ca894ee5c143ffbfc46130d1d677c5 2024-11-13T05:41:26,784 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/00ca894ee5c143ffbfc46130d1d677c5, entries=150, sequenceid=196, filesize=11.9 K 2024-11-13T05:41:26,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/b8eec58646ba4f009eb4fc065580d4ea as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/b8eec58646ba4f009eb4fc065580d4ea 2024-11-13T05:41:26,791 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/b8eec58646ba4f009eb4fc065580d4ea, entries=150, sequenceid=196, filesize=11.9 K 2024-11-13T05:41:26,792 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for cebd6d1afed0df6f58e14d0610d86767 in 543ms, sequenceid=196, compaction requested=false 2024-11-13T05:41:26,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:26,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:26,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-13T05:41:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-13T05:41:26,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-13T05:41:26,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1550 sec 2024-11-13T05:41:26,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.1620 sec 2024-11-13T05:41:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:26,892 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:26,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T05:41:26,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bc19ca9021a943f8b5e3db39fefa97c3_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476486547/Put/seqid=0 2024-11-13T05:41:26,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742221_1397 (size=14794) 2024-11-13T05:41:26,911 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:26,915 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bc19ca9021a943f8b5e3db39fefa97c3_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bc19ca9021a943f8b5e3db39fefa97c3_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:26,915 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/0bc29fb221a74fab9d7a0fa693fd2b3f, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:26,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/0bc29fb221a74fab9d7a0fa693fd2b3f is 175, key is test_row_0/A:col10/1731476486547/Put/seqid=0 2024-11-13T05:41:26,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742222_1398 (size=39749) 2024-11-13T05:41:26,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476546951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:26,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476546954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:26,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476546963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:27,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476547067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:27,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:27,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476547076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:27,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:27,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476547086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:27,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:27,131 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/846b794be6414835a74422a25aaafe08 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/846b794be6414835a74422a25aaafe08 2024-11-13T05:41:27,136 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/A of cebd6d1afed0df6f58e14d0610d86767 into 846b794be6414835a74422a25aaafe08(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:27,136 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:27,136 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/A, priority=13, startTime=1731476486165; duration=0sec 2024-11-13T05:41:27,137 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:27,137 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:A 2024-11-13T05:41:27,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476547287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:27,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476547296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:27,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476547296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:27,319 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/0bc29fb221a74fab9d7a0fa693fd2b3f 2024-11-13T05:41:27,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/c9a2051db5af488bbf296b55c2f8e07d is 50, key is test_row_0/B:col10/1731476486547/Put/seqid=0 2024-11-13T05:41:27,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742223_1399 (size=12151) 2024-11-13T05:41:27,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476547606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:27,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476547617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:27,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476547617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:27,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/c9a2051db5af488bbf296b55c2f8e07d 2024-11-13T05:41:27,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/393c61e7cf4d498ca1bdadb50ac9a145 is 50, key is test_row_0/C:col10/1731476486547/Put/seqid=0 2024-11-13T05:41:27,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742224_1400 (size=12151) 2024-11-13T05:41:27,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/393c61e7cf4d498ca1bdadb50ac9a145 2024-11-13T05:41:27,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/0bc29fb221a74fab9d7a0fa693fd2b3f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0bc29fb221a74fab9d7a0fa693fd2b3f 2024-11-13T05:41:27,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0bc29fb221a74fab9d7a0fa693fd2b3f, entries=200, sequenceid=214, filesize=38.8 K 2024-11-13T05:41:27,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/c9a2051db5af488bbf296b55c2f8e07d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c9a2051db5af488bbf296b55c2f8e07d 2024-11-13T05:41:27,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c9a2051db5af488bbf296b55c2f8e07d, entries=150, sequenceid=214, filesize=11.9 K 2024-11-13T05:41:27,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/393c61e7cf4d498ca1bdadb50ac9a145 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/393c61e7cf4d498ca1bdadb50ac9a145 2024-11-13T05:41:27,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/393c61e7cf4d498ca1bdadb50ac9a145, entries=150, sequenceid=214, filesize=11.9 K 2024-11-13T05:41:27,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for cebd6d1afed0df6f58e14d0610d86767 in 866ms, sequenceid=214, compaction requested=true 2024-11-13T05:41:27,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:27,758 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:27,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:27,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:27,758 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:27,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:27,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:27,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:27,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:27,759 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:27,759 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/A is initiating minor compaction (all files) 2024-11-13T05:41:27,759 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/A in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:27,759 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:27,759 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/B is initiating minor compaction (all files) 2024-11-13T05:41:27,759 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/846b794be6414835a74422a25aaafe08, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b52bcaeab95e4d498056e9323d00b582, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0bc29fb221a74fab9d7a0fa693fd2b3f] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=100.0 K 2024-11-13T05:41:27,759 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/B in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:27,759 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/e2dc68cf8a994e27ba5a0fbc4a002abf, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/00ca894ee5c143ffbfc46130d1d677c5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c9a2051db5af488bbf296b55c2f8e07d] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=36.0 K 2024-11-13T05:41:27,759 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:27,759 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/846b794be6414835a74422a25aaafe08, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b52bcaeab95e4d498056e9323d00b582, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0bc29fb221a74fab9d7a0fa693fd2b3f] 2024-11-13T05:41:27,760 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting e2dc68cf8a994e27ba5a0fbc4a002abf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731476484132 2024-11-13T05:41:27,760 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 846b794be6414835a74422a25aaafe08, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731476484132 2024-11-13T05:41:27,760 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 00ca894ee5c143ffbfc46130d1d677c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731476485349 2024-11-13T05:41:27,760 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting b52bcaeab95e4d498056e9323d00b582, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731476485349 2024-11-13T05:41:27,760 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting c9a2051db5af488bbf296b55c2f8e07d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731476486547 2024-11-13T05:41:27,760 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc29fb221a74fab9d7a0fa693fd2b3f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731476486547 2024-11-13T05:41:27,768 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:27,769 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:27,769 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/4d64ace0c5f84e10b69faf9020939dc5 is 50, key is test_row_0/B:col10/1731476486547/Put/seqid=0 2024-11-13T05:41:27,771 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111336dc873c527c4d9fb43b6dda4f48b7c5_cebd6d1afed0df6f58e14d0610d86767 store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:27,773 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111336dc873c527c4d9fb43b6dda4f48b7c5_cebd6d1afed0df6f58e14d0610d86767, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:27,773 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111336dc873c527c4d9fb43b6dda4f48b7c5_cebd6d1afed0df6f58e14d0610d86767 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:27,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742225_1401 (size=12663) 2024-11-13T05:41:27,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742226_1402 (size=4469) 2024-11-13T05:41:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T05:41:27,786 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:27,787 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:27,787 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#A#compaction#337 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:27,787 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/c5618ff9d2a345c793d48ff547b3564c is 175, key is test_row_0/A:col10/1731476486547/Put/seqid=0 2024-11-13T05:41:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-13T05:41:27,788 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:27,789 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:27,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T05:41:27,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742227_1403 (size=31617) 2024-11-13T05:41:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T05:41:27,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-13T05:41:27,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:27,942 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T05:41:27,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a7e66f33b7a94be086511fe40096c5e8_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476486942/Put/seqid=0 2024-11-13T05:41:27,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742228_1404 (size=12304) 2024-11-13T05:41:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T05:41:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:28,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:28,184 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/4d64ace0c5f84e10b69faf9020939dc5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/4d64ace0c5f84e10b69faf9020939dc5 2024-11-13T05:41:28,190 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/B of cebd6d1afed0df6f58e14d0610d86767 into 4d64ace0c5f84e10b69faf9020939dc5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:28,190 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:28,190 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/B, priority=13, startTime=1731476487758; duration=0sec 2024-11-13T05:41:28,190 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:28,190 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:B 2024-11-13T05:41:28,191 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:28,191 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:28,191 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/C is initiating minor compaction (all files) 2024-11-13T05:41:28,192 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/C in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:28,192 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/77e1c83912fd45fda2ddbcfec7b80c40, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/b8eec58646ba4f009eb4fc065580d4ea, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/393c61e7cf4d498ca1bdadb50ac9a145] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=36.0 K 2024-11-13T05:41:28,192 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 77e1c83912fd45fda2ddbcfec7b80c40, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731476484132 2024-11-13T05:41:28,192 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting b8eec58646ba4f009eb4fc065580d4ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731476485349 2024-11-13T05:41:28,193 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 393c61e7cf4d498ca1bdadb50ac9a145, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731476486547 2024-11-13T05:41:28,199 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#C#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:28,200 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/7874639ab3ec4c2c8fd20182def47e38 is 50, key is test_row_0/C:col10/1731476486547/Put/seqid=0 2024-11-13T05:41:28,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476548188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476548189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:28,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476548203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:28,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:28,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742229_1405 (size=12663) 2024-11-13T05:41:28,207 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/c5618ff9d2a345c793d48ff547b3564c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/c5618ff9d2a345c793d48ff547b3564c 2024-11-13T05:41:28,212 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/A of cebd6d1afed0df6f58e14d0610d86767 into c5618ff9d2a345c793d48ff547b3564c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:28,212 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:28,212 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/A, priority=13, startTime=1731476487758; duration=0sec 2024-11-13T05:41:28,212 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:28,212 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:A 2024-11-13T05:41:28,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476548306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476548306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:28,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476548316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:28,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:28,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:28,359 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a7e66f33b7a94be086511fe40096c5e8_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a7e66f33b7a94be086511fe40096c5e8_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:28,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/04ba6a5bfe0e424085fd83f4bc56d87d, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:28,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/04ba6a5bfe0e424085fd83f4bc56d87d is 175, key is test_row_0/A:col10/1731476486942/Put/seqid=0 2024-11-13T05:41:28,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742230_1406 (size=31105) 2024-11-13T05:41:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T05:41:28,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476548516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:28,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476548517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:28,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476548526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:28,611 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/7874639ab3ec4c2c8fd20182def47e38 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/7874639ab3ec4c2c8fd20182def47e38 2024-11-13T05:41:28,614 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/C of cebd6d1afed0df6f58e14d0610d86767 into 7874639ab3ec4c2c8fd20182def47e38(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:28,614 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:28,614 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/C, priority=13, startTime=1731476487758; duration=0sec 2024-11-13T05:41:28,615 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:28,615 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:C 2024-11-13T05:41:28,765 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/04ba6a5bfe0e424085fd83f4bc56d87d 2024-11-13T05:41:28,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/8c6fab90c934431bab5b21d5833e7ee1 is 50, key is test_row_0/B:col10/1731476486942/Put/seqid=0 2024-11-13T05:41:28,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742231_1407 (size=12151) 2024-11-13T05:41:28,774 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/8c6fab90c934431bab5b21d5833e7ee1 2024-11-13T05:41:28,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/81f30aa0904f426aa0196b40f9f65871 is 50, key is test_row_0/C:col10/1731476486942/Put/seqid=0 2024-11-13T05:41:28,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742232_1408 (size=12151) 2024-11-13T05:41:28,792 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/81f30aa0904f426aa0196b40f9f65871 2024-11-13T05:41:28,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/04ba6a5bfe0e424085fd83f4bc56d87d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/04ba6a5bfe0e424085fd83f4bc56d87d 2024-11-13T05:41:28,801 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/04ba6a5bfe0e424085fd83f4bc56d87d, entries=150, sequenceid=235, filesize=30.4 K 2024-11-13T05:41:28,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/8c6fab90c934431bab5b21d5833e7ee1 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/8c6fab90c934431bab5b21d5833e7ee1 2024-11-13T05:41:28,811 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/8c6fab90c934431bab5b21d5833e7ee1, entries=150, sequenceid=235, filesize=11.9 K 2024-11-13T05:41:28,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/81f30aa0904f426aa0196b40f9f65871 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/81f30aa0904f426aa0196b40f9f65871 2024-11-13T05:41:28,824 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/81f30aa0904f426aa0196b40f9f65871, entries=150, sequenceid=235, filesize=11.9 K 2024-11-13T05:41:28,825 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for cebd6d1afed0df6f58e14d0610d86767 in 883ms, sequenceid=235, compaction requested=false 2024-11-13T05:41:28,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:28,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:28,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-13T05:41:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-13T05:41:28,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-13T05:41:28,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0370 sec 2024-11-13T05:41:28,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.0410 sec 2024-11-13T05:41:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:28,841 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:28,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T05:41:28,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134ddde894fe624301b9ec2916a4262d4f_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476488841/Put/seqid=0 2024-11-13T05:41:28,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742233_1409 (size=14794) 2024-11-13T05:41:28,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476548883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:28,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:28,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476548884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:28,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:28,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476548885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-13T05:41:28,926 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:28,926 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-13T05:41:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T05:41:28,928 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:28,929 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:28,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:28,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476548996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:28,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476548997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:29,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476549006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:29,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T05:41:29,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T05:41:29,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:29,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:29,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:29,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476549207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476549207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:29,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:29,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T05:41:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476549226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:29,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:29,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:29,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:29,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T05:41:29,253 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:29,257 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134ddde894fe624301b9ec2916a4262d4f_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134ddde894fe624301b9ec2916a4262d4f_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:29,257 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/f1346c0020b54d72a06bce0188f670a0, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:29,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/f1346c0020b54d72a06bce0188f670a0 is 175, key is test_row_0/A:col10/1731476488841/Put/seqid=0 2024-11-13T05:41:29,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742234_1410 (size=39749) 2024-11-13T05:41:29,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T05:41:29,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:29,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:29,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:29,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476549517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:29,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476549526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:29,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:29,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T05:41:29,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:29,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:29,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:29,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476549546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T05:41:29,669 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/f1346c0020b54d72a06bce0188f670a0 2024-11-13T05:41:29,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/a7dade30aa53461e96c40fe7fb8424da is 50, key is test_row_0/B:col10/1731476488841/Put/seqid=0 2024-11-13T05:41:29,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742235_1411 (size=12151) 2024-11-13T05:41:29,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T05:41:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:29,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:29,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:29,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731476549766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:29,768 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in cebd6d1afed0df6f58e14d0610d86767 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:29,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60242 deadline: 1731476549786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:29,789 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_0' in cebd6d1afed0df6f58e14d0610d86767 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:29,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:29,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:29,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T05:41:29,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:29,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:29,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:29,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:29,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T05:41:29,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:29,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:30,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:30,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:30,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:30,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476550036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:30,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476550046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:30,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:30,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:30,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T05:41:30,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476550066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:30,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/a7dade30aa53461e96c40fe7fb8424da 2024-11-13T05:41:30,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/79efe936184e4ca7b3786058420758b4 is 50, key is test_row_0/C:col10/1731476488841/Put/seqid=0 2024-11-13T05:41:30,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742236_1412 (size=12151) 2024-11-13T05:41:30,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/79efe936184e4ca7b3786058420758b4 2024-11-13T05:41:30,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/f1346c0020b54d72a06bce0188f670a0 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/f1346c0020b54d72a06bce0188f670a0 2024-11-13T05:41:30,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/f1346c0020b54d72a06bce0188f670a0, entries=200, sequenceid=256, filesize=38.8 K 2024-11-13T05:41:30,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/a7dade30aa53461e96c40fe7fb8424da as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/a7dade30aa53461e96c40fe7fb8424da 2024-11-13T05:41:30,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/a7dade30aa53461e96c40fe7fb8424da, entries=150, sequenceid=256, filesize=11.9 K 2024-11-13T05:41:30,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/79efe936184e4ca7b3786058420758b4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/79efe936184e4ca7b3786058420758b4 2024-11-13T05:41:30,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/79efe936184e4ca7b3786058420758b4, entries=150, sequenceid=256, filesize=11.9 K 2024-11-13T05:41:30,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for cebd6d1afed0df6f58e14d0610d86767 in 1270ms, sequenceid=256, compaction requested=true 2024-11-13T05:41:30,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:30,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:30,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:30,112 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:30,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:30,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:30,112 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:30,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:30,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:30,112 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:30,112 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:30,112 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/A is initiating minor compaction (all files) 2024-11-13T05:41:30,112 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/B is initiating minor compaction (all files) 2024-11-13T05:41:30,112 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/A in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:30,112 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/B in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:30,112 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/4d64ace0c5f84e10b69faf9020939dc5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/8c6fab90c934431bab5b21d5833e7ee1, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/a7dade30aa53461e96c40fe7fb8424da] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=36.1 K 2024-11-13T05:41:30,112 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/c5618ff9d2a345c793d48ff547b3564c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/04ba6a5bfe0e424085fd83f4bc56d87d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/f1346c0020b54d72a06bce0188f670a0] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=100.1 K 2024-11-13T05:41:30,112 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:30,112 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/c5618ff9d2a345c793d48ff547b3564c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/04ba6a5bfe0e424085fd83f4bc56d87d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/f1346c0020b54d72a06bce0188f670a0] 2024-11-13T05:41:30,114 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d64ace0c5f84e10b69faf9020939dc5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731476486547 2024-11-13T05:41:30,114 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5618ff9d2a345c793d48ff547b3564c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731476486547 2024-11-13T05:41:30,114 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04ba6a5bfe0e424085fd83f4bc56d87d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731476486942 2024-11-13T05:41:30,114 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c6fab90c934431bab5b21d5833e7ee1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731476486942 2024-11-13T05:41:30,115 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1346c0020b54d72a06bce0188f670a0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731476488187 2024-11-13T05:41:30,115 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting a7dade30aa53461e96c40fe7fb8424da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731476488187 2024-11-13T05:41:30,118 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:30,120 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113da535a2f36d14e069d075d79b856ca68_cebd6d1afed0df6f58e14d0610d86767 store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:30,121 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#B#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:30,121 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/cd47c00cea5445c8a9b29d7a61aaf7d2 is 50, key is test_row_0/B:col10/1731476488841/Put/seqid=0 2024-11-13T05:41:30,122 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113da535a2f36d14e069d075d79b856ca68_cebd6d1afed0df6f58e14d0610d86767, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:30,122 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113da535a2f36d14e069d075d79b856ca68_cebd6d1afed0df6f58e14d0610d86767 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:30,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742238_1414 (size=4469) 2024-11-13T05:41:30,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742237_1413 (size=12765) 2024-11-13T05:41:30,141 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/cd47c00cea5445c8a9b29d7a61aaf7d2 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/cd47c00cea5445c8a9b29d7a61aaf7d2 2024-11-13T05:41:30,144 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/B of cebd6d1afed0df6f58e14d0610d86767 into cd47c00cea5445c8a9b29d7a61aaf7d2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:30,144 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:30,144 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/B, priority=13, startTime=1731476490112; duration=0sec 2024-11-13T05:41:30,144 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:30,144 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:B 2024-11-13T05:41:30,144 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:30,146 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:30,147 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/C is initiating minor compaction (all files) 2024-11-13T05:41:30,147 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/C in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:30,147 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/7874639ab3ec4c2c8fd20182def47e38, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/81f30aa0904f426aa0196b40f9f65871, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/79efe936184e4ca7b3786058420758b4] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=36.1 K 2024-11-13T05:41:30,147 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 7874639ab3ec4c2c8fd20182def47e38, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731476486547 2024-11-13T05:41:30,147 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 81f30aa0904f426aa0196b40f9f65871, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731476486942 2024-11-13T05:41:30,147 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 79efe936184e4ca7b3786058420758b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731476488187 2024-11-13T05:41:30,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T05:41:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:30,152 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-13T05:41:30,154 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#C#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:30,154 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/37b407213f6f483093a31c2cdf73e321 is 50, key is test_row_0/C:col10/1731476488841/Put/seqid=0 2024-11-13T05:41:30,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136a218066c21b47ea8acd69a6f96f6e74_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476488883/Put/seqid=0 2024-11-13T05:41:30,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742239_1415 (size=12765) 2024-11-13T05:41:30,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742240_1416 (size=12454) 2024-11-13T05:41:30,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:30,168 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/37b407213f6f483093a31c2cdf73e321 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/37b407213f6f483093a31c2cdf73e321 2024-11-13T05:41:30,170 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136a218066c21b47ea8acd69a6f96f6e74_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136a218066c21b47ea8acd69a6f96f6e74_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:30,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/4ebdf50aeae6418180a508db6cc5a3ce, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:30,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/4ebdf50aeae6418180a508db6cc5a3ce is 175, key is test_row_0/A:col10/1731476488883/Put/seqid=0 2024-11-13T05:41:30,174 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/C of cebd6d1afed0df6f58e14d0610d86767 into 37b407213f6f483093a31c2cdf73e321(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:30,174 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:30,174 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/C, priority=13, startTime=1731476490112; duration=0sec 2024-11-13T05:41:30,174 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:30,174 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:C 2024-11-13T05:41:30,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742241_1417 (size=31255) 2024-11-13T05:41:30,533 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#A#compaction#345 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:30,533 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/8256446ec7554d708c1dd509d3c869eb is 175, key is test_row_0/A:col10/1731476488841/Put/seqid=0 2024-11-13T05:41:30,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742242_1418 (size=31719) 2024-11-13T05:41:30,578 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/4ebdf50aeae6418180a508db6cc5a3ce 2024-11-13T05:41:30,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/9dbf5ba5476749c5a00a0b689bb95cc1 is 50, key is test_row_0/B:col10/1731476488883/Put/seqid=0 2024-11-13T05:41:30,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742243_1419 (size=12301) 2024-11-13T05:41:30,941 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/8256446ec7554d708c1dd509d3c869eb as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/8256446ec7554d708c1dd509d3c869eb 2024-11-13T05:41:30,944 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/A of cebd6d1afed0df6f58e14d0610d86767 into 8256446ec7554d708c1dd509d3c869eb(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:30,944 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:30,944 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/A, priority=13, startTime=1731476490111; duration=0sec 2024-11-13T05:41:30,945 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:30,945 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:A 2024-11-13T05:41:30,988 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/9dbf5ba5476749c5a00a0b689bb95cc1 2024-11-13T05:41:30,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/297e63ecc13f4398bfd29f79ebccda72 is 50, key is test_row_0/C:col10/1731476488883/Put/seqid=0 2024-11-13T05:41:31,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742244_1420 (size=12301) 2024-11-13T05:41:31,000 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/297e63ecc13f4398bfd29f79ebccda72 2024-11-13T05:41:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/4ebdf50aeae6418180a508db6cc5a3ce as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/4ebdf50aeae6418180a508db6cc5a3ce 2024-11-13T05:41:31,007 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/4ebdf50aeae6418180a508db6cc5a3ce, entries=150, sequenceid=276, filesize=30.5 K 2024-11-13T05:41:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/9dbf5ba5476749c5a00a0b689bb95cc1 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/9dbf5ba5476749c5a00a0b689bb95cc1 2024-11-13T05:41:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,013 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/9dbf5ba5476749c5a00a0b689bb95cc1, entries=150, sequenceid=276, filesize=12.0 K 2024-11-13T05:41:31,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/297e63ecc13f4398bfd29f79ebccda72 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/297e63ecc13f4398bfd29f79ebccda72 2024-11-13T05:41:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,017 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/297e63ecc13f4398bfd29f79ebccda72, entries=150, sequenceid=276, filesize=12.0 K 2024-11-13T05:41:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,018 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for cebd6d1afed0df6f58e14d0610d86767 in 866ms, sequenceid=276, compaction requested=false 2024-11-13T05:41:31,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:31,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:31,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-13T05:41:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-13T05:41:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-13T05:41:31,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0900 sec 2024-11-13T05:41:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,023 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.0960 sec 2024-11-13T05:41:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T05:41:31,077 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,077 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-13T05:41:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T05:41:31,079 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,079 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:31,177 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:31,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:41:31,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113070acc06a2c842d4839b833a04af7bf4_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476491177/Put/seqid=0 2024-11-13T05:41:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T05:41:31,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742245_1421 (size=20074) 2024-11-13T05:41:31,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T05:41:31,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:31,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:31,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:31,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476551231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:31,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476551241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:31,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:31,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476551242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:31,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476551347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:31,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476551347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:31,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:31,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:31,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476551356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:31,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:31,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T05:41:31,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:31,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:31,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:31,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T05:41:31,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T05:41:31,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:31,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:31,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:31,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476551556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:31,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476551556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:31,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476551566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:31,596 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:31,599 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113070acc06a2c842d4839b833a04af7bf4_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113070acc06a2c842d4839b833a04af7bf4_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:31,600 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/72ff164c73e146b38fa619b9a5049e5b, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:31,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/72ff164c73e146b38fa619b9a5049e5b is 175, key is test_row_0/A:col10/1731476491177/Put/seqid=0 2024-11-13T05:41:31,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742246_1422 (size=57333) 2024-11-13T05:41:31,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T05:41:31,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:31,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:31,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:31,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T05:41:31,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T05:41:31,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:31,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:31,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:31,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:31,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476551877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:31,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:31,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476551877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:31,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:31,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:31,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:31,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476551877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:31,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:31,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:31,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T05:41:31,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:31,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:31,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:31,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,008 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/72ff164c73e146b38fa619b9a5049e5b 2024-11-13T05:41:32,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/76f7c6f948f6495c849595589455e371 is 50, key is test_row_0/B:col10/1731476491177/Put/seqid=0 2024-11-13T05:41:32,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742247_1423 (size=12301) 2024-11-13T05:41:32,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T05:41:32,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:32,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:32,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:32,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T05:41:32,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T05:41:32,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:32,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:32,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:32,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:32,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476552387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:32,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:32,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:32,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:32,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476552387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:32,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:32,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:32,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:32,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:32,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476552396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:32,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/76f7c6f948f6495c849595589455e371 2024-11-13T05:41:32,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/d170c6b17f1a4df2aab86524d1930101 is 50, key is test_row_0/C:col10/1731476491177/Put/seqid=0 2024-11-13T05:41:32,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742248_1424 (size=12301) 2024-11-13T05:41:32,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T05:41:32,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:32,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:32,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:32,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T05:41:32,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:32,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:32,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:32,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T05:41:32,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:32,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:32,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:32,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:32,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/d170c6b17f1a4df2aab86524d1930101 2024-11-13T05:41:32,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/72ff164c73e146b38fa619b9a5049e5b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/72ff164c73e146b38fa619b9a5049e5b 2024-11-13T05:41:32,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/72ff164c73e146b38fa619b9a5049e5b, entries=300, sequenceid=290, filesize=56.0 K 2024-11-13T05:41:32,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/76f7c6f948f6495c849595589455e371 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/76f7c6f948f6495c849595589455e371 2024-11-13T05:41:32,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/76f7c6f948f6495c849595589455e371, entries=150, sequenceid=290, filesize=12.0 K 2024-11-13T05:41:32,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/d170c6b17f1a4df2aab86524d1930101 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/d170c6b17f1a4df2aab86524d1930101 2024-11-13T05:41:32,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/d170c6b17f1a4df2aab86524d1930101, entries=150, sequenceid=290, filesize=12.0 K 2024-11-13T05:41:32,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for cebd6d1afed0df6f58e14d0610d86767 in 1660ms, sequenceid=290, compaction requested=true 2024-11-13T05:41:32,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:32,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:32,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:32,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:32,838 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:32,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:32,838 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:32,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:32,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:32,839 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120307 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:32,839 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:32,839 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/B is initiating minor compaction (all files) 2024-11-13T05:41:32,839 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/A is initiating minor compaction (all files) 2024-11-13T05:41:32,839 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/B in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:32,839 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/A in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:32,839 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/8256446ec7554d708c1dd509d3c869eb, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/4ebdf50aeae6418180a508db6cc5a3ce, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/72ff164c73e146b38fa619b9a5049e5b] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=117.5 K 2024-11-13T05:41:32,839 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/cd47c00cea5445c8a9b29d7a61aaf7d2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/9dbf5ba5476749c5a00a0b689bb95cc1, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/76f7c6f948f6495c849595589455e371] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=36.5 K 2024-11-13T05:41:32,839 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:32,839 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/8256446ec7554d708c1dd509d3c869eb, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/4ebdf50aeae6418180a508db6cc5a3ce, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/72ff164c73e146b38fa619b9a5049e5b] 2024-11-13T05:41:32,839 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting cd47c00cea5445c8a9b29d7a61aaf7d2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731476488187 2024-11-13T05:41:32,840 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8256446ec7554d708c1dd509d3c869eb, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731476488187 2024-11-13T05:41:32,840 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dbf5ba5476749c5a00a0b689bb95cc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731476488863 2024-11-13T05:41:32,840 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f7c6f948f6495c849595589455e371, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731476491163 2024-11-13T05:41:32,840 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ebdf50aeae6418180a508db6cc5a3ce, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731476488863 2024-11-13T05:41:32,840 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72ff164c73e146b38fa619b9a5049e5b, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731476491163 2024-11-13T05:41:32,845 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:32,846 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:32,846 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/9a854fa4600a4bd68031cd32ee7e31e8 is 50, key is test_row_0/B:col10/1731476491177/Put/seqid=0 2024-11-13T05:41:32,847 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113ba14fc632950404e858cda3df2bc60ce_cebd6d1afed0df6f58e14d0610d86767 store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:32,849 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113ba14fc632950404e858cda3df2bc60ce_cebd6d1afed0df6f58e14d0610d86767, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:32,849 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ba14fc632950404e858cda3df2bc60ce_cebd6d1afed0df6f58e14d0610d86767 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:32,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742249_1425 (size=13017) 2024-11-13T05:41:32,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742250_1426 (size=4469) 2024-11-13T05:41:32,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T05:41:32,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:32,914 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T05:41:32,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113aac8016fa85547c59b4e7b8953d31e73_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476491240/Put/seqid=0 2024-11-13T05:41:32,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742251_1427 (size=12454) 2024-11-13T05:41:33,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T05:41:33,259 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#A#compaction#355 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:33,259 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/c98473b2725b4a428e973f921ddf9c19 is 175, key is test_row_0/A:col10/1731476491177/Put/seqid=0 2024-11-13T05:41:33,260 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/9a854fa4600a4bd68031cd32ee7e31e8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/9a854fa4600a4bd68031cd32ee7e31e8 2024-11-13T05:41:33,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742252_1428 (size=31971) 2024-11-13T05:41:33,267 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/c98473b2725b4a428e973f921ddf9c19 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/c98473b2725b4a428e973f921ddf9c19 2024-11-13T05:41:33,267 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/B of cebd6d1afed0df6f58e14d0610d86767 into 9a854fa4600a4bd68031cd32ee7e31e8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:33,267 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:33,267 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/B, priority=13, startTime=1731476492838; duration=0sec 2024-11-13T05:41:33,267 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:33,267 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:B 2024-11-13T05:41:33,267 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:33,268 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:33,268 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/C is initiating minor compaction (all files) 2024-11-13T05:41:33,268 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/C in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:33,268 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/37b407213f6f483093a31c2cdf73e321, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/297e63ecc13f4398bfd29f79ebccda72, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/d170c6b17f1a4df2aab86524d1930101] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=36.5 K 2024-11-13T05:41:33,269 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 37b407213f6f483093a31c2cdf73e321, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731476488187 2024-11-13T05:41:33,269 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 297e63ecc13f4398bfd29f79ebccda72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731476488863 2024-11-13T05:41:33,270 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting d170c6b17f1a4df2aab86524d1930101, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731476491163 2024-11-13T05:41:33,271 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/A of cebd6d1afed0df6f58e14d0610d86767 into c98473b2725b4a428e973f921ddf9c19(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:33,271 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:33,271 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/A, priority=13, startTime=1731476492838; duration=0sec 2024-11-13T05:41:33,272 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:33,272 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:A 2024-11-13T05:41:33,277 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#C#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:33,278 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/d4d4696987984828a594f8d5b8d7bc8d is 50, key is test_row_0/C:col10/1731476491177/Put/seqid=0 2024-11-13T05:41:33,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742253_1429 (size=13017) 2024-11-13T05:41:33,290 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/d4d4696987984828a594f8d5b8d7bc8d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/d4d4696987984828a594f8d5b8d7bc8d 2024-11-13T05:41:33,294 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/C of cebd6d1afed0df6f58e14d0610d86767 into d4d4696987984828a594f8d5b8d7bc8d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:33,294 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:33,294 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/C, priority=13, startTime=1731476492838; duration=0sec 2024-11-13T05:41:33,294 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:33,294 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:C 2024-11-13T05:41:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:33,327 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113aac8016fa85547c59b4e7b8953d31e73_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113aac8016fa85547c59b4e7b8953d31e73_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:33,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b75e5bf2d1ba4c7bbdd752ef2269e486, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:33,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b75e5bf2d1ba4c7bbdd752ef2269e486 is 175, key is test_row_0/A:col10/1731476491240/Put/seqid=0 2024-11-13T05:41:33,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742254_1430 (size=31255) 2024-11-13T05:41:33,339 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b75e5bf2d1ba4c7bbdd752ef2269e486 2024-11-13T05:41:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/6533d36019634a9e9c88680642162d64 is 50, key is test_row_0/B:col10/1731476491240/Put/seqid=0 2024-11-13T05:41:33,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742255_1431 (size=12301) 2024-11-13T05:41:33,357 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/6533d36019634a9e9c88680642162d64 2024-11-13T05:41:33,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/5055a08d922b4131b09df574eff67460 is 50, key is test_row_0/C:col10/1731476491240/Put/seqid=0 2024-11-13T05:41:33,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742256_1432 (size=12301) 2024-11-13T05:41:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:33,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. as already flushing 2024-11-13T05:41:33,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476553422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:33,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:33,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476553423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:33,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:33,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476553424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:33,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476553537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:33,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476553537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476553537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:33,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476553746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476553746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476553747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:33,769 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/5055a08d922b4131b09df574eff67460 2024-11-13T05:41:33,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/b75e5bf2d1ba4c7bbdd752ef2269e486 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b75e5bf2d1ba4c7bbdd752ef2269e486 2024-11-13T05:41:33,784 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b75e5bf2d1ba4c7bbdd752ef2269e486, entries=150, sequenceid=314, filesize=30.5 K 2024-11-13T05:41:33,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/6533d36019634a9e9c88680642162d64 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/6533d36019634a9e9c88680642162d64 2024-11-13T05:41:33,788 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/6533d36019634a9e9c88680642162d64, entries=150, sequenceid=314, filesize=12.0 K 2024-11-13T05:41:33,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/5055a08d922b4131b09df574eff67460 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/5055a08d922b4131b09df574eff67460 2024-11-13T05:41:33,793 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/5055a08d922b4131b09df574eff67460, entries=150, sequenceid=314, filesize=12.0 K 2024-11-13T05:41:33,794 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for cebd6d1afed0df6f58e14d0610d86767 in 879ms, sequenceid=314, compaction requested=false 2024-11-13T05:41:33,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:33,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:33,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-13T05:41:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-13T05:41:33,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-13T05:41:33,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7160 sec 2024-11-13T05:41:33,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.7210 sec 2024-11-13T05:41:34,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:34,063 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:34,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T05:41:34,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134126faab73754b40b21984385850d4e1_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476493420/Put/seqid=0 2024-11-13T05:41:34,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742257_1433 (size=17534) 2024-11-13T05:41:34,073 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:34,076 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134126faab73754b40b21984385850d4e1_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134126faab73754b40b21984385850d4e1_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:34,077 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/0d912618b88c4498b1d50fa2bba5f392, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:34,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/0d912618b88c4498b1d50fa2bba5f392 is 175, key is test_row_0/A:col10/1731476493420/Put/seqid=0 2024-11-13T05:41:34,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742258_1434 (size=48639) 2024-11-13T05:41:34,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476554119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:34,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476554119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:34,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476554120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:34,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:34,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476554227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:34,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476554236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:34,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476554236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:34,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476554436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:34,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476554447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:34,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476554447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:34,481 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/0d912618b88c4498b1d50fa2bba5f392 2024-11-13T05:41:34,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/2c4bbeb79aad49b598334e6bfd533077 is 50, key is test_row_0/B:col10/1731476493420/Put/seqid=0 2024-11-13T05:41:34,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742259_1435 (size=12301) 2024-11-13T05:41:34,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731476554746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:34,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60196 deadline: 1731476554756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:34,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60220 deadline: 1731476554766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:41:34,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/2c4bbeb79aad49b598334e6bfd533077 2024-11-13T05:41:34,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/633bb7b5fac34c6ca7a06cee233e066b is 50, key is test_row_0/C:col10/1731476493420/Put/seqid=0 2024-11-13T05:41:34,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742260_1436 (size=12301) 2024-11-13T05:41:34,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/633bb7b5fac34c6ca7a06cee233e066b 2024-11-13T05:41:34,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/0d912618b88c4498b1d50fa2bba5f392 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0d912618b88c4498b1d50fa2bba5f392 2024-11-13T05:41:34,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0d912618b88c4498b1d50fa2bba5f392, entries=250, sequenceid=332, filesize=47.5 K 2024-11-13T05:41:34,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/2c4bbeb79aad49b598334e6bfd533077 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/2c4bbeb79aad49b598334e6bfd533077 2024-11-13T05:41:34,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/2c4bbeb79aad49b598334e6bfd533077, entries=150, sequenceid=332, filesize=12.0 K 2024-11-13T05:41:34,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/633bb7b5fac34c6ca7a06cee233e066b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/633bb7b5fac34c6ca7a06cee233e066b 2024-11-13T05:41:34,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/633bb7b5fac34c6ca7a06cee233e066b, entries=150, sequenceid=332, filesize=12.0 K 2024-11-13T05:41:34,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for cebd6d1afed0df6f58e14d0610d86767 in 877ms, sequenceid=332, compaction requested=true 2024-11-13T05:41:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:34,940 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:34,940 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:34,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:34,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:34,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cebd6d1afed0df6f58e14d0610d86767:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:34,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:34,942 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:34,942 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/B is initiating minor compaction (all files) 2024-11-13T05:41:34,942 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/B in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:34,942 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/9a854fa4600a4bd68031cd32ee7e31e8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/6533d36019634a9e9c88680642162d64, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/2c4bbeb79aad49b598334e6bfd533077] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=36.7 K 2024-11-13T05:41:34,943 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:34,943 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/A is initiating minor compaction (all files) 2024-11-13T05:41:34,944 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/A in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:34,944 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/c98473b2725b4a428e973f921ddf9c19, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b75e5bf2d1ba4c7bbdd752ef2269e486, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0d912618b88c4498b1d50fa2bba5f392] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=109.2 K 2024-11-13T05:41:34,944 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:34,944 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/c98473b2725b4a428e973f921ddf9c19, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b75e5bf2d1ba4c7bbdd752ef2269e486, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0d912618b88c4498b1d50fa2bba5f392] 2024-11-13T05:41:34,944 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a854fa4600a4bd68031cd32ee7e31e8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731476491163 2024-11-13T05:41:34,944 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting c98473b2725b4a428e973f921ddf9c19, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731476491163 2024-11-13T05:41:34,944 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 6533d36019634a9e9c88680642162d64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731476491226 2024-11-13T05:41:34,944 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c4bbeb79aad49b598334e6bfd533077, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731476493416 2024-11-13T05:41:34,944 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting b75e5bf2d1ba4c7bbdd752ef2269e486, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731476491226 2024-11-13T05:41:34,945 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d912618b88c4498b1d50fa2bba5f392, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731476493416 2024-11-13T05:41:34,964 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:34,965 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#B#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:34,965 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/546daedc94d649e69aee2c69e79ea992 is 50, key is test_row_0/B:col10/1731476493420/Put/seqid=0 2024-11-13T05:41:34,975 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411139aa8ece621124bea9f7e5c5432d58451_cebd6d1afed0df6f58e14d0610d86767 store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:34,989 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411139aa8ece621124bea9f7e5c5432d58451_cebd6d1afed0df6f58e14d0610d86767, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:34,989 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139aa8ece621124bea9f7e5c5432d58451_cebd6d1afed0df6f58e14d0610d86767 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:34,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742261_1437 (size=13119) 2024-11-13T05:41:35,006 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/546daedc94d649e69aee2c69e79ea992 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/546daedc94d649e69aee2c69e79ea992 2024-11-13T05:41:35,016 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/B of cebd6d1afed0df6f58e14d0610d86767 into 546daedc94d649e69aee2c69e79ea992(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:35,016 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:35,016 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/B, priority=13, startTime=1731476494940; duration=0sec 2024-11-13T05:41:35,016 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:35,016 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:B 2024-11-13T05:41:35,017 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:35,018 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:35,018 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): cebd6d1afed0df6f58e14d0610d86767/C is initiating minor compaction (all files) 2024-11-13T05:41:35,019 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cebd6d1afed0df6f58e14d0610d86767/C in TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:35,019 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/d4d4696987984828a594f8d5b8d7bc8d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/5055a08d922b4131b09df574eff67460, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/633bb7b5fac34c6ca7a06cee233e066b] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp, totalSize=36.7 K 2024-11-13T05:41:35,019 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting d4d4696987984828a594f8d5b8d7bc8d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731476491163 2024-11-13T05:41:35,019 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 5055a08d922b4131b09df574eff67460, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731476491226 2024-11-13T05:41:35,019 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 633bb7b5fac34c6ca7a06cee233e066b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731476493416 2024-11-13T05:41:35,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742262_1438 (size=4469) 2024-11-13T05:41:35,027 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#C#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:35,028 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/0f40414d886643f386c03d8f5147b2c8 is 50, key is test_row_0/C:col10/1731476493420/Put/seqid=0 2024-11-13T05:41:35,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742263_1439 (size=13119) 2024-11-13T05:41:35,039 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/0f40414d886643f386c03d8f5147b2c8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/0f40414d886643f386c03d8f5147b2c8 2024-11-13T05:41:35,043 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/C of cebd6d1afed0df6f58e14d0610d86767 into 0f40414d886643f386c03d8f5147b2c8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:35,043 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:35,043 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/C, priority=13, startTime=1731476494941; duration=0sec 2024-11-13T05:41:35,043 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:35,043 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:C 2024-11-13T05:41:35,150 INFO [Thread-1565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1565. 2024-11-13T05:41:35,150 INFO [Thread-1557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1557. 2024-11-13T05:41:35,150 DEBUG [Thread-1557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:35,150 DEBUG [Thread-1565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:35,150 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,150 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,150 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,150 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,150 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:35,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:35,152 INFO [Thread-1563 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1563. 2024-11-13T05:41:35,152 DEBUG [Thread-1563 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:35,152 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,152 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,152 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:35,156 INFO [Thread-1561 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1561. 2024-11-13T05:41:35,156 DEBUG [Thread-1561 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:35,156 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,156 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,156 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:35,156 INFO [Thread-1559 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1559. 2024-11-13T05:41:35,156 DEBUG [Thread-1559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:35,156 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,156 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,156 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T05:41:35,238 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:35,271 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:35,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T05:41:35,271 INFO [Thread-1546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1546. 2024-11-13T05:41:35,271 DEBUG [Thread-1546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:35,271 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,272 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:35,280 INFO [Thread-1548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1548. 2024-11-13T05:41:35,280 DEBUG [Thread-1548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:35,280 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,280 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:35,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111308a4098cf3bb4593b660a5e79410d941_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476495268/Put/seqid=0 2024-11-13T05:41:35,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742264_1440 (size=12454) 2024-11-13T05:41:35,287 INFO [Thread-1552 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1552. 2024-11-13T05:41:35,287 DEBUG [Thread-1552 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:35,287 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,287 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:35,287 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:35,430 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cebd6d1afed0df6f58e14d0610d86767#A#compaction#364 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:35,432 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/5ba3ebcca79343f9ae5cf6f0b309b7a4 is 175, key is test_row_0/A:col10/1731476493420/Put/seqid=0 2024-11-13T05:41:35,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742265_1441 (size=32073) 2024-11-13T05:41:35,687 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:35,696 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111308a4098cf3bb4593b660a5e79410d941_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111308a4098cf3bb4593b660a5e79410d941_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:35,698 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/d788c4f20cb643b28d5d10dc025ad871, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:35,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/d788c4f20cb643b28d5d10dc025ad871 is 175, key is test_row_0/A:col10/1731476495268/Put/seqid=0 2024-11-13T05:41:35,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742266_1442 (size=31255) 2024-11-13T05:41:35,848 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/5ba3ebcca79343f9ae5cf6f0b309b7a4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/5ba3ebcca79343f9ae5cf6f0b309b7a4 2024-11-13T05:41:35,855 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cebd6d1afed0df6f58e14d0610d86767/A of cebd6d1afed0df6f58e14d0610d86767 into 5ba3ebcca79343f9ae5cf6f0b309b7a4(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:35,855 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:35,855 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., storeName=cebd6d1afed0df6f58e14d0610d86767/A, priority=13, startTime=1731476494940; duration=0sec 2024-11-13T05:41:35,855 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:35,855 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cebd6d1afed0df6f58e14d0610d86767:A 2024-11-13T05:41:36,105 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/d788c4f20cb643b28d5d10dc025ad871 2024-11-13T05:41:36,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/48cd3fd65b224a7b92db210d78b4040e is 50, key is test_row_0/B:col10/1731476495268/Put/seqid=0 2024-11-13T05:41:36,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742267_1443 (size=12301) 2024-11-13T05:41:36,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/48cd3fd65b224a7b92db210d78b4040e 2024-11-13T05:41:36,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/24456b3ce14244daadc37d2f28dc2992 is 50, key is test_row_0/C:col10/1731476495268/Put/seqid=0 2024-11-13T05:41:36,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742268_1444 (size=12301) 2024-11-13T05:41:36,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/24456b3ce14244daadc37d2f28dc2992 2024-11-13T05:41:36,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/d788c4f20cb643b28d5d10dc025ad871 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/d788c4f20cb643b28d5d10dc025ad871 2024-11-13T05:41:36,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/d788c4f20cb643b28d5d10dc025ad871, entries=150, sequenceid=356, filesize=30.5 K 2024-11-13T05:41:36,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/48cd3fd65b224a7b92db210d78b4040e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/48cd3fd65b224a7b92db210d78b4040e 2024-11-13T05:41:36,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/48cd3fd65b224a7b92db210d78b4040e, entries=150, sequenceid=356, filesize=12.0 K 2024-11-13T05:41:36,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/24456b3ce14244daadc37d2f28dc2992 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/24456b3ce14244daadc37d2f28dc2992 2024-11-13T05:41:36,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/24456b3ce14244daadc37d2f28dc2992, entries=150, sequenceid=356, filesize=12.0 K 2024-11-13T05:41:36,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=13.42 KB/13740 for cebd6d1afed0df6f58e14d0610d86767 in 1684ms, sequenceid=356, compaction requested=false 2024-11-13T05:41:36,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cebd6d1afed0df6f58e14d0610d86767: 2024-11-13T05:41:38,392 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:41:39,808 INFO [Thread-1554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1554. 2024-11-13T05:41:39,808 DEBUG [Thread-1554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:39,808 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:39,808 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:39,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:39,845 INFO [Thread-1550 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1550. 2024-11-13T05:41:39,845 DEBUG [Thread-1550 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:41:39,845 DEBUG [Thread-1550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:39,845 DEBUG [Thread-1550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:39,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:39,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1332 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3996 rows 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1339 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4014 rows 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1339 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4014 rows 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1344 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4026 rows 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1340 2024-11-13T05:41:39,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4015 rows 2024-11-13T05:41:39,846 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:41:39,846 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:41:39,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:39,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:39,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:41:39,847 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T05:41:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T05:41:39,849 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476499849"}]},"ts":"1731476499849"} 2024-11-13T05:41:39,850 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T05:41:39,850 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T05:41:39,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:41:39,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, UNASSIGN}] 2024-11-13T05:41:39,853 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, UNASSIGN 2024-11-13T05:41:39,853 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=101 updating hbase:meta row=cebd6d1afed0df6f58e14d0610d86767, regionState=CLOSING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:39,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, UNASSIGN because future has completed 2024-11-13T05:41:39,855 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:41:39,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; CloseRegionProcedure cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:41:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T05:41:40,008 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(122): Close cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:40,008 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:41:40,008 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1722): Closing cebd6d1afed0df6f58e14d0610d86767, disabling compactions & flushes 2024-11-13T05:41:40,008 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:40,008 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:40,008 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. after waiting 0 ms 2024-11-13T05:41:40,008 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:40,009 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(2902): Flushing cebd6d1afed0df6f58e14d0610d86767 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-13T05:41:40,020 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131d642facabc544ad92d7f1acc51c0d3d_cebd6d1afed0df6f58e14d0610d86767 is 50, key is test_row_0/A:col10/1731476499806/Put/seqid=0 2024-11-13T05:41:40,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742269_1445 (size=12454) 2024-11-13T05:41:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T05:41:40,424 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:40,434 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131d642facabc544ad92d7f1acc51c0d3d_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131d642facabc544ad92d7f1acc51c0d3d_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:40,434 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/12997c30831a4548b212a7a3f2f56eb7, store: [table=TestAcidGuarantees family=A region=cebd6d1afed0df6f58e14d0610d86767] 2024-11-13T05:41:40,435 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/12997c30831a4548b212a7a3f2f56eb7 is 175, key is test_row_0/A:col10/1731476499806/Put/seqid=0 2024-11-13T05:41:40,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742270_1446 (size=31255) 2024-11-13T05:41:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T05:41:40,839 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/12997c30831a4548b212a7a3f2f56eb7 2024-11-13T05:41:40,851 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/ed13c51ba9de445d8527d6d427d4ef9d is 50, key is test_row_0/B:col10/1731476499806/Put/seqid=0 2024-11-13T05:41:40,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742271_1447 (size=12301) 2024-11-13T05:41:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T05:41:41,258 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/ed13c51ba9de445d8527d6d427d4ef9d 2024-11-13T05:41:41,272 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/18c828de43d1415584e1aac367f6106c is 50, key is test_row_0/C:col10/1731476499806/Put/seqid=0 2024-11-13T05:41:41,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742272_1448 (size=12301) 2024-11-13T05:41:41,676 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/18c828de43d1415584e1aac367f6106c 2024-11-13T05:41:41,679 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/A/12997c30831a4548b212a7a3f2f56eb7 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/12997c30831a4548b212a7a3f2f56eb7 2024-11-13T05:41:41,682 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/12997c30831a4548b212a7a3f2f56eb7, entries=150, sequenceid=364, filesize=30.5 K 2024-11-13T05:41:41,683 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/B/ed13c51ba9de445d8527d6d427d4ef9d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/ed13c51ba9de445d8527d6d427d4ef9d 2024-11-13T05:41:41,685 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/ed13c51ba9de445d8527d6d427d4ef9d, entries=150, sequenceid=364, filesize=12.0 K 2024-11-13T05:41:41,686 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/.tmp/C/18c828de43d1415584e1aac367f6106c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/18c828de43d1415584e1aac367f6106c 2024-11-13T05:41:41,689 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/18c828de43d1415584e1aac367f6106c, entries=150, sequenceid=364, filesize=12.0 K 2024-11-13T05:41:41,690 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for cebd6d1afed0df6f58e14d0610d86767 in 1682ms, sequenceid=364, compaction requested=true 2024-11-13T05:41:41,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/da0f12681b024747a2f7f1667612dc08, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/51c778673c374336ab956c3be8daebda, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/1e7a903a77ed43ee9bdfe5a0755f76b3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/dd69c76b2e7141a1b784edf1832090a1, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/97ec014b77f44c2ea3fc594ed211f390, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b93f9774f6f048e997f46aa682798d78, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/49d0bcb399f44b6389bebb920855e9c4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0ea64484836545f1b5ebedd99ae8f56f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/1b94be91aef94a609faa94ec1282376f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/cc37ca1a31a64e1eb7997fe3ab90ca2a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/072ddde985984d818e8a54d1dd91324a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b2cccc31abc14edd9f3bdff86a7fc6fa, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/846b794be6414835a74422a25aaafe08, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b52bcaeab95e4d498056e9323d00b582, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0bc29fb221a74fab9d7a0fa693fd2b3f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/c5618ff9d2a345c793d48ff547b3564c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/04ba6a5bfe0e424085fd83f4bc56d87d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/f1346c0020b54d72a06bce0188f670a0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/8256446ec7554d708c1dd509d3c869eb, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/4ebdf50aeae6418180a508db6cc5a3ce, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/72ff164c73e146b38fa619b9a5049e5b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/c98473b2725b4a428e973f921ddf9c19, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b75e5bf2d1ba4c7bbdd752ef2269e486, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0d912618b88c4498b1d50fa2bba5f392] to archive 2024-11-13T05:41:41,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:41:41,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/da0f12681b024747a2f7f1667612dc08 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/da0f12681b024747a2f7f1667612dc08 2024-11-13T05:41:41,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/51c778673c374336ab956c3be8daebda to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/51c778673c374336ab956c3be8daebda 2024-11-13T05:41:41,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/1e7a903a77ed43ee9bdfe5a0755f76b3 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/1e7a903a77ed43ee9bdfe5a0755f76b3 2024-11-13T05:41:41,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/dd69c76b2e7141a1b784edf1832090a1 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/dd69c76b2e7141a1b784edf1832090a1 2024-11-13T05:41:41,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/97ec014b77f44c2ea3fc594ed211f390 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/97ec014b77f44c2ea3fc594ed211f390 2024-11-13T05:41:41,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b93f9774f6f048e997f46aa682798d78 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b93f9774f6f048e997f46aa682798d78 2024-11-13T05:41:41,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/49d0bcb399f44b6389bebb920855e9c4 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/49d0bcb399f44b6389bebb920855e9c4 2024-11-13T05:41:41,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0ea64484836545f1b5ebedd99ae8f56f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0ea64484836545f1b5ebedd99ae8f56f 2024-11-13T05:41:41,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/1b94be91aef94a609faa94ec1282376f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/1b94be91aef94a609faa94ec1282376f 2024-11-13T05:41:41,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/cc37ca1a31a64e1eb7997fe3ab90ca2a to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/cc37ca1a31a64e1eb7997fe3ab90ca2a 2024-11-13T05:41:41,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/072ddde985984d818e8a54d1dd91324a to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/072ddde985984d818e8a54d1dd91324a 2024-11-13T05:41:41,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b2cccc31abc14edd9f3bdff86a7fc6fa to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b2cccc31abc14edd9f3bdff86a7fc6fa 2024-11-13T05:41:41,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/846b794be6414835a74422a25aaafe08 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/846b794be6414835a74422a25aaafe08 2024-11-13T05:41:41,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b52bcaeab95e4d498056e9323d00b582 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b52bcaeab95e4d498056e9323d00b582 2024-11-13T05:41:41,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0bc29fb221a74fab9d7a0fa693fd2b3f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0bc29fb221a74fab9d7a0fa693fd2b3f 2024-11-13T05:41:41,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/c5618ff9d2a345c793d48ff547b3564c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/c5618ff9d2a345c793d48ff547b3564c 2024-11-13T05:41:41,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/04ba6a5bfe0e424085fd83f4bc56d87d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/04ba6a5bfe0e424085fd83f4bc56d87d 2024-11-13T05:41:41,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/f1346c0020b54d72a06bce0188f670a0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/f1346c0020b54d72a06bce0188f670a0 2024-11-13T05:41:41,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/8256446ec7554d708c1dd509d3c869eb to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/8256446ec7554d708c1dd509d3c869eb 2024-11-13T05:41:41,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/4ebdf50aeae6418180a508db6cc5a3ce to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/4ebdf50aeae6418180a508db6cc5a3ce 2024-11-13T05:41:41,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/72ff164c73e146b38fa619b9a5049e5b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/72ff164c73e146b38fa619b9a5049e5b 2024-11-13T05:41:41,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/c98473b2725b4a428e973f921ddf9c19 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/c98473b2725b4a428e973f921ddf9c19 2024-11-13T05:41:41,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b75e5bf2d1ba4c7bbdd752ef2269e486 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/b75e5bf2d1ba4c7bbdd752ef2269e486 2024-11-13T05:41:41,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0d912618b88c4498b1d50fa2bba5f392 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/0d912618b88c4498b1d50fa2bba5f392 2024-11-13T05:41:41,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/40c0b2dd3fb643bc8110a3c9737a15a7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/6b8a6c68adae47faa1b97048437f0560, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c0551510c20849719111acbbb85992c9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/3301d2eddcfc4acdb6e61e2d8223b84c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/d72a7d1027fb4af399312236d3247fa5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/0288c04bb54443d6b511829c7a1e983a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/25f2e98ef26e483aab25f6be07d23f74, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/00a4cf7ee70b4d3586e843f5af3f50b2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/a709864c2e3f40a9a3128db5c0706395, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/fe679ebfa1794f0c85eacbb52e56c366, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c676e48441d740b2812e0f09813b537e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/e2dc68cf8a994e27ba5a0fbc4a002abf, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/0b52b8dc89d742e19ede3e70a3323473, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/00ca894ee5c143ffbfc46130d1d677c5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/4d64ace0c5f84e10b69faf9020939dc5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c9a2051db5af488bbf296b55c2f8e07d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/8c6fab90c934431bab5b21d5833e7ee1, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/cd47c00cea5445c8a9b29d7a61aaf7d2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/a7dade30aa53461e96c40fe7fb8424da, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/9dbf5ba5476749c5a00a0b689bb95cc1, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/9a854fa4600a4bd68031cd32ee7e31e8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/76f7c6f948f6495c849595589455e371, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/6533d36019634a9e9c88680642162d64, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/2c4bbeb79aad49b598334e6bfd533077] to archive 2024-11-13T05:41:41,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:41:41,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/40c0b2dd3fb643bc8110a3c9737a15a7 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/40c0b2dd3fb643bc8110a3c9737a15a7 2024-11-13T05:41:41,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/6b8a6c68adae47faa1b97048437f0560 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/6b8a6c68adae47faa1b97048437f0560 2024-11-13T05:41:41,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c0551510c20849719111acbbb85992c9 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c0551510c20849719111acbbb85992c9 2024-11-13T05:41:41,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/3301d2eddcfc4acdb6e61e2d8223b84c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/3301d2eddcfc4acdb6e61e2d8223b84c 2024-11-13T05:41:41,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/d72a7d1027fb4af399312236d3247fa5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/d72a7d1027fb4af399312236d3247fa5 2024-11-13T05:41:41,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/0288c04bb54443d6b511829c7a1e983a to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/0288c04bb54443d6b511829c7a1e983a 2024-11-13T05:41:41,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/25f2e98ef26e483aab25f6be07d23f74 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/25f2e98ef26e483aab25f6be07d23f74 2024-11-13T05:41:41,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/00a4cf7ee70b4d3586e843f5af3f50b2 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/00a4cf7ee70b4d3586e843f5af3f50b2 2024-11-13T05:41:41,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/a709864c2e3f40a9a3128db5c0706395 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/a709864c2e3f40a9a3128db5c0706395 2024-11-13T05:41:41,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/fe679ebfa1794f0c85eacbb52e56c366 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/fe679ebfa1794f0c85eacbb52e56c366 2024-11-13T05:41:41,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c676e48441d740b2812e0f09813b537e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c676e48441d740b2812e0f09813b537e 2024-11-13T05:41:41,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/e2dc68cf8a994e27ba5a0fbc4a002abf to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/e2dc68cf8a994e27ba5a0fbc4a002abf 2024-11-13T05:41:41,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/0b52b8dc89d742e19ede3e70a3323473 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/0b52b8dc89d742e19ede3e70a3323473 2024-11-13T05:41:41,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/00ca894ee5c143ffbfc46130d1d677c5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/00ca894ee5c143ffbfc46130d1d677c5 2024-11-13T05:41:41,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/4d64ace0c5f84e10b69faf9020939dc5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/4d64ace0c5f84e10b69faf9020939dc5 2024-11-13T05:41:41,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c9a2051db5af488bbf296b55c2f8e07d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/c9a2051db5af488bbf296b55c2f8e07d 2024-11-13T05:41:41,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/8c6fab90c934431bab5b21d5833e7ee1 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/8c6fab90c934431bab5b21d5833e7ee1 2024-11-13T05:41:41,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/cd47c00cea5445c8a9b29d7a61aaf7d2 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/cd47c00cea5445c8a9b29d7a61aaf7d2 2024-11-13T05:41:41,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/a7dade30aa53461e96c40fe7fb8424da to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/a7dade30aa53461e96c40fe7fb8424da 2024-11-13T05:41:41,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/9dbf5ba5476749c5a00a0b689bb95cc1 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/9dbf5ba5476749c5a00a0b689bb95cc1 2024-11-13T05:41:41,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/9a854fa4600a4bd68031cd32ee7e31e8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/9a854fa4600a4bd68031cd32ee7e31e8 2024-11-13T05:41:41,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/76f7c6f948f6495c849595589455e371 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/76f7c6f948f6495c849595589455e371 2024-11-13T05:41:41,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/6533d36019634a9e9c88680642162d64 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/6533d36019634a9e9c88680642162d64 2024-11-13T05:41:41,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/2c4bbeb79aad49b598334e6bfd533077 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/2c4bbeb79aad49b598334e6bfd533077 2024-11-13T05:41:41,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/175b99d715584e9ba4328f41b3f8be4f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/44d42e151c12473c8d845694971fca5a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/aed3607597934e69a140486011ce193b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/f8df0a2e9d4b4bd5bc1032643d930cea, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/40b3bfbf072a4a9e8c19f5d6a21da298, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/8371a3ae534140dda27079b7d6191577, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/3b5f913a93c24668a4de670935457bd7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/05343cd96fd94b32b6d4d6a087f718cc, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/6c49085414b34ad0aed809d2a546b125, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/4d928a00f6dd4b589b9d15a90c37d75c, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/390a15d25cd3470b8f1ac44fdb510f2f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/77e1c83912fd45fda2ddbcfec7b80c40, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/4beb4a2a5dd54e80b86ff6de97de8a21, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/b8eec58646ba4f009eb4fc065580d4ea, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/7874639ab3ec4c2c8fd20182def47e38, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/393c61e7cf4d498ca1bdadb50ac9a145, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/81f30aa0904f426aa0196b40f9f65871, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/37b407213f6f483093a31c2cdf73e321, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/79efe936184e4ca7b3786058420758b4, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/297e63ecc13f4398bfd29f79ebccda72, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/d4d4696987984828a594f8d5b8d7bc8d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/d170c6b17f1a4df2aab86524d1930101, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/5055a08d922b4131b09df574eff67460, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/633bb7b5fac34c6ca7a06cee233e066b] to archive 2024-11-13T05:41:41,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:41:41,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/175b99d715584e9ba4328f41b3f8be4f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/175b99d715584e9ba4328f41b3f8be4f 2024-11-13T05:41:41,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/44d42e151c12473c8d845694971fca5a to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/44d42e151c12473c8d845694971fca5a 2024-11-13T05:41:41,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/aed3607597934e69a140486011ce193b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/aed3607597934e69a140486011ce193b 2024-11-13T05:41:41,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/f8df0a2e9d4b4bd5bc1032643d930cea to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/f8df0a2e9d4b4bd5bc1032643d930cea 2024-11-13T05:41:41,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/40b3bfbf072a4a9e8c19f5d6a21da298 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/40b3bfbf072a4a9e8c19f5d6a21da298 2024-11-13T05:41:41,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/8371a3ae534140dda27079b7d6191577 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/8371a3ae534140dda27079b7d6191577 2024-11-13T05:41:41,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/3b5f913a93c24668a4de670935457bd7 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/3b5f913a93c24668a4de670935457bd7 2024-11-13T05:41:41,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/05343cd96fd94b32b6d4d6a087f718cc to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/05343cd96fd94b32b6d4d6a087f718cc 2024-11-13T05:41:41,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/6c49085414b34ad0aed809d2a546b125 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/6c49085414b34ad0aed809d2a546b125 2024-11-13T05:41:41,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/4d928a00f6dd4b589b9d15a90c37d75c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/4d928a00f6dd4b589b9d15a90c37d75c 2024-11-13T05:41:41,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/390a15d25cd3470b8f1ac44fdb510f2f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/390a15d25cd3470b8f1ac44fdb510f2f 2024-11-13T05:41:41,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/77e1c83912fd45fda2ddbcfec7b80c40 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/77e1c83912fd45fda2ddbcfec7b80c40 2024-11-13T05:41:41,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/4beb4a2a5dd54e80b86ff6de97de8a21 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/4beb4a2a5dd54e80b86ff6de97de8a21 2024-11-13T05:41:41,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/b8eec58646ba4f009eb4fc065580d4ea to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/b8eec58646ba4f009eb4fc065580d4ea 2024-11-13T05:41:41,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/7874639ab3ec4c2c8fd20182def47e38 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/7874639ab3ec4c2c8fd20182def47e38 2024-11-13T05:41:41,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/393c61e7cf4d498ca1bdadb50ac9a145 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/393c61e7cf4d498ca1bdadb50ac9a145 2024-11-13T05:41:41,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/81f30aa0904f426aa0196b40f9f65871 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/81f30aa0904f426aa0196b40f9f65871 2024-11-13T05:41:41,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/37b407213f6f483093a31c2cdf73e321 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/37b407213f6f483093a31c2cdf73e321 2024-11-13T05:41:41,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/79efe936184e4ca7b3786058420758b4 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/79efe936184e4ca7b3786058420758b4 2024-11-13T05:41:41,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/297e63ecc13f4398bfd29f79ebccda72 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/297e63ecc13f4398bfd29f79ebccda72 2024-11-13T05:41:41,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/d4d4696987984828a594f8d5b8d7bc8d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/d4d4696987984828a594f8d5b8d7bc8d 2024-11-13T05:41:41,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/d170c6b17f1a4df2aab86524d1930101 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/d170c6b17f1a4df2aab86524d1930101 2024-11-13T05:41:41,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/5055a08d922b4131b09df574eff67460 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/5055a08d922b4131b09df574eff67460 2024-11-13T05:41:41,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/633bb7b5fac34c6ca7a06cee233e066b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/633bb7b5fac34c6ca7a06cee233e066b 2024-11-13T05:41:41,778 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/recovered.edits/367.seqid, newMaxSeqId=367, maxSeqId=4 2024-11-13T05:41:41,779 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. 2024-11-13T05:41:41,779 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1676): Region close journal for cebd6d1afed0df6f58e14d0610d86767: Waiting for close lock at 1731476500008Running coprocessor pre-close hooks at 1731476500008Disabling compacts and flushes for region at 1731476500008Disabling writes for close at 1731476500008Obtaining lock to block concurrent updates at 1731476500009 (+1 ms)Preparing flush snapshotting stores in cebd6d1afed0df6f58e14d0610d86767 at 1731476500009Finished memstore snapshotting TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731476500009Flushing stores of TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767. at 1731476500011 (+2 ms)Flushing cebd6d1afed0df6f58e14d0610d86767/A: creating writer at 1731476500011Flushing mob file cebd6d1afed0df6f58e14d0610d86767/A: appending metadata at 1731476500019 (+8 ms)Flushing mob file cebd6d1afed0df6f58e14d0610d86767/A: closing flushed file at 1731476500019Flushing cebd6d1afed0df6f58e14d0610d86767/A: appending metadata at 1731476500434 (+415 ms)Flushing cebd6d1afed0df6f58e14d0610d86767/A: closing flushed file at 1731476500435 (+1 ms)Flushing cebd6d1afed0df6f58e14d0610d86767/B: creating writer at 1731476500846 (+411 ms)Flushing cebd6d1afed0df6f58e14d0610d86767/B: appending metadata at 1731476500851 (+5 ms)Flushing cebd6d1afed0df6f58e14d0610d86767/B: closing flushed file at 1731476500851Flushing cebd6d1afed0df6f58e14d0610d86767/C: creating writer at 1731476501267 (+416 ms)Flushing cebd6d1afed0df6f58e14d0610d86767/C: appending metadata at 1731476501271 (+4 ms)Flushing cebd6d1afed0df6f58e14d0610d86767/C: closing flushed file at 1731476501271Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e606f67: reopening flushed file at 1731476501679 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f4468ca: reopening flushed file at 1731476501682 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65a1c1af: reopening flushed file at 1731476501685 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for cebd6d1afed0df6f58e14d0610d86767 in 1682ms, sequenceid=364, compaction requested=true at 1731476501690 (+5 ms)Writing region close event to WAL at 1731476501775 (+85 ms)Running coprocessor post-close hooks at 1731476501779 (+4 ms)Closed at 1731476501779 2024-11-13T05:41:41,781 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(157): Closed cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:41,781 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=101 updating hbase:meta row=cebd6d1afed0df6f58e14d0610d86767, regionState=CLOSED 2024-11-13T05:41:41,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE, hasLock=false; CloseRegionProcedure cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:41:41,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-13T05:41:41,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseRegionProcedure cebd6d1afed0df6f58e14d0610d86767, server=bc386b7bf2fe,45837,1731476380128 in 1.9290 sec 2024-11-13T05:41:41,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-13T05:41:41,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cebd6d1afed0df6f58e14d0610d86767, UNASSIGN in 1.9340 sec 2024-11-13T05:41:41,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-13T05:41:41,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9370 sec 2024-11-13T05:41:41,791 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476501791"}]},"ts":"1731476501791"} 2024-11-13T05:41:41,793 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T05:41:41,793 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T05:41:41,795 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9470 sec 2024-11-13T05:41:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T05:41:41,998 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:42,000 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T05:41:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:42,004 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=103, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T05:41:42,005 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=103, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:42,007 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,010 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/recovered.edits] 2024-11-13T05:41:42,013 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/12997c30831a4548b212a7a3f2f56eb7 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/12997c30831a4548b212a7a3f2f56eb7 2024-11-13T05:41:42,015 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/5ba3ebcca79343f9ae5cf6f0b309b7a4 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/5ba3ebcca79343f9ae5cf6f0b309b7a4 2024-11-13T05:41:42,016 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/d788c4f20cb643b28d5d10dc025ad871 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/A/d788c4f20cb643b28d5d10dc025ad871 2024-11-13T05:41:42,019 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/48cd3fd65b224a7b92db210d78b4040e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/48cd3fd65b224a7b92db210d78b4040e 2024-11-13T05:41:42,021 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/546daedc94d649e69aee2c69e79ea992 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/546daedc94d649e69aee2c69e79ea992 2024-11-13T05:41:42,022 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/ed13c51ba9de445d8527d6d427d4ef9d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/B/ed13c51ba9de445d8527d6d427d4ef9d 2024-11-13T05:41:42,024 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/0f40414d886643f386c03d8f5147b2c8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/0f40414d886643f386c03d8f5147b2c8 2024-11-13T05:41:42,024 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/18c828de43d1415584e1aac367f6106c to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/18c828de43d1415584e1aac367f6106c 2024-11-13T05:41:42,025 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/24456b3ce14244daadc37d2f28dc2992 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/C/24456b3ce14244daadc37d2f28dc2992 2024-11-13T05:41:42,026 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/recovered.edits/367.seqid to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767/recovered.edits/367.seqid 2024-11-13T05:41:42,027 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,027 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T05:41:42,027 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T05:41:42,028 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T05:41:42,029 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113070acc06a2c842d4839b833a04af7bf4_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113070acc06a2c842d4839b833a04af7bf4_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,030 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111308a4098cf3bb4593b660a5e79410d941_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111308a4098cf3bb4593b660a5e79410d941_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,031 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131d642facabc544ad92d7f1acc51c0d3d_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131d642facabc544ad92d7f1acc51c0d3d_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,031 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131e6a457b002649ceb9d9a98d4a285a52_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131e6a457b002649ceb9d9a98d4a285a52_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,032 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134126faab73754b40b21984385850d4e1_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134126faab73754b40b21984385850d4e1_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,033 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134ddde894fe624301b9ec2916a4262d4f_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134ddde894fe624301b9ec2916a4262d4f_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,034 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111358a12772241d4664b939d167524988a6_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111358a12772241d4664b939d167524988a6_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,034 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135a29b2d9f7a04b019e27a10f3dcf6994_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135a29b2d9f7a04b019e27a10f3dcf6994_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,035 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135b935f4971744200bbd3aafde5a1efad_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411135b935f4971744200bbd3aafde5a1efad_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,036 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136a218066c21b47ea8acd69a6f96f6e74_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136a218066c21b47ea8acd69a6f96f6e74_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,037 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136e20cf262ec74664a7e858bf4f10c0b1_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136e20cf262ec74664a7e858bf4f10c0b1_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,038 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139f3ec59c893a459ba3476fd96cd7be4a_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139f3ec59c893a459ba3476fd96cd7be4a_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,039 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139fdf03ce5baf496f8f95ebbc2dee178d_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139fdf03ce5baf496f8f95ebbc2dee178d_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,040 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a019128cd14d45d7aaa68eaf8ece1d12_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a019128cd14d45d7aaa68eaf8ece1d12_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,041 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a7e66f33b7a94be086511fe40096c5e8_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a7e66f33b7a94be086511fe40096c5e8_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,041 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113aac8016fa85547c59b4e7b8953d31e73_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113aac8016fa85547c59b4e7b8953d31e73_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,042 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bc19ca9021a943f8b5e3db39fefa97c3_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bc19ca9021a943f8b5e3db39fefa97c3_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,043 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bea6bd7f5759469a904c74a5dd8d340c_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bea6bd7f5759469a904c74a5dd8d340c_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,044 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ca0adf5b778b4b8787dda616e5a0afad_cebd6d1afed0df6f58e14d0610d86767 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ca0adf5b778b4b8787dda616e5a0afad_cebd6d1afed0df6f58e14d0610d86767 2024-11-13T05:41:42,045 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T05:41:42,046 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=103, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:42,048 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T05:41:42,050 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T05:41:42,051 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=103, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:42,051 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T05:41:42,051 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731476502051"}]},"ts":"9223372036854775807"} 2024-11-13T05:41:42,053 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T05:41:42,053 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cebd6d1afed0df6f58e14d0610d86767, NAME => 'TestAcidGuarantees,,1731476471968.cebd6d1afed0df6f58e14d0610d86767.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T05:41:42,053 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T05:41:42,053 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731476502053"}]},"ts":"9223372036854775807"} 2024-11-13T05:41:42,054 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T05:41:42,055 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=103, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:42,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-11-13T05:41:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T05:41:42,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T05:41:42,122 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:42,130 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237 (was 241), OpenFileDescriptor=453 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=454 (was 477), ProcessCount=11 (was 11), AvailableMemoryMB=7279 (was 6831) - AvailableMemoryMB LEAK? - 2024-11-13T05:41:42,138 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=454, ProcessCount=11, AvailableMemoryMB=7279 2024-11-13T05:41:42,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:41:42,139 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:41:42,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T05:41:42,141 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:41:42,141 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:42,141 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 104 2024-11-13T05:41:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-13T05:41:42,142 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:41:42,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742273_1449 (size=959) 2024-11-13T05:41:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-13T05:41:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-13T05:41:42,549 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745 2024-11-13T05:41:42,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742274_1450 (size=53) 2024-11-13T05:41:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-13T05:41:42,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:41:42,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 666f8f0c32525e1ac64394208a6558b0, disabling compactions & flushes 2024-11-13T05:41:42,959 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:42,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:42,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. after waiting 0 ms 2024-11-13T05:41:42,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:42,959 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:42,959 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 666f8f0c32525e1ac64394208a6558b0: Waiting for close lock at 1731476502959Disabling compacts and flushes for region at 1731476502959Disabling writes for close at 1731476502959Writing region close event to WAL at 1731476502959Closed at 1731476502959 2024-11-13T05:41:42,962 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:41:42,962 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731476502962"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731476502962"}]},"ts":"1731476502962"} 2024-11-13T05:41:42,965 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:41:42,966 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:41:42,967 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476502967"}]},"ts":"1731476502967"} 2024-11-13T05:41:42,969 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T05:41:42,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=666f8f0c32525e1ac64394208a6558b0, ASSIGN}] 2024-11-13T05:41:42,971 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=666f8f0c32525e1ac64394208a6558b0, ASSIGN 2024-11-13T05:41:42,972 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=666f8f0c32525e1ac64394208a6558b0, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45837,1731476380128; forceNewPlan=false, retain=false 2024-11-13T05:41:43,123 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=666f8f0c32525e1ac64394208a6558b0, regionState=OPENING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:43,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=666f8f0c32525e1ac64394208a6558b0, ASSIGN because future has completed 2024-11-13T05:41:43,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; OpenRegionProcedure 666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:41:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-13T05:41:43,280 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:43,280 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7752): Opening region: {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:41:43,281 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:43,281 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:41:43,281 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7794): checking encryption for 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:43,281 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7797): checking classloading for 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:43,282 INFO [StoreOpener-666f8f0c32525e1ac64394208a6558b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:43,284 INFO [StoreOpener-666f8f0c32525e1ac64394208a6558b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 666f8f0c32525e1ac64394208a6558b0 columnFamilyName A 2024-11-13T05:41:43,284 DEBUG [StoreOpener-666f8f0c32525e1ac64394208a6558b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:43,284 INFO [StoreOpener-666f8f0c32525e1ac64394208a6558b0-1 {}] regionserver.HStore(327): Store=666f8f0c32525e1ac64394208a6558b0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:41:43,285 INFO [StoreOpener-666f8f0c32525e1ac64394208a6558b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:43,286 INFO [StoreOpener-666f8f0c32525e1ac64394208a6558b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 666f8f0c32525e1ac64394208a6558b0 columnFamilyName B 2024-11-13T05:41:43,286 DEBUG [StoreOpener-666f8f0c32525e1ac64394208a6558b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:43,286 INFO [StoreOpener-666f8f0c32525e1ac64394208a6558b0-1 {}] regionserver.HStore(327): Store=666f8f0c32525e1ac64394208a6558b0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:41:43,287 INFO [StoreOpener-666f8f0c32525e1ac64394208a6558b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:43,288 INFO [StoreOpener-666f8f0c32525e1ac64394208a6558b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 666f8f0c32525e1ac64394208a6558b0 columnFamilyName C 2024-11-13T05:41:43,288 DEBUG [StoreOpener-666f8f0c32525e1ac64394208a6558b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:41:43,288 INFO [StoreOpener-666f8f0c32525e1ac64394208a6558b0-1 {}] regionserver.HStore(327): Store=666f8f0c32525e1ac64394208a6558b0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:41:43,288 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1038): replaying wal for 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:43,289 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:43,290 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:43,291 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1048): stopping wal replay for 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:43,291 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1060): Cleaning up temporary data for 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:43,291 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:41:43,292 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): writing seq id for 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:43,293 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:41:43,294 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1114): Opened 666f8f0c32525e1ac64394208a6558b0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73598270, jitterRate=0.09669968485832214}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:41:43,294 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:43,294 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1006): Region open journal for 666f8f0c32525e1ac64394208a6558b0: Running coprocessor pre-open hook at 1731476503281Writing region info on filesystem at 1731476503281Initializing all the Stores at 1731476503282 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476503282Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476503282Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476503282Cleaning up temporary data from old regions at 1731476503291 (+9 ms)Running coprocessor post-open hooks at 1731476503294 (+3 ms)Region opened successfully at 1731476503294 2024-11-13T05:41:43,295 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., pid=106, masterSystemTime=1731476503277 2024-11-13T05:41:43,296 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:43,296 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:43,297 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=666f8f0c32525e1ac64394208a6558b0, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:43,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; OpenRegionProcedure 666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:41:43,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-13T05:41:43,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; OpenRegionProcedure 666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 in 174 msec 2024-11-13T05:41:43,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-13T05:41:43,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=666f8f0c32525e1ac64394208a6558b0, ASSIGN in 331 msec 2024-11-13T05:41:43,302 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:41:43,302 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476503302"}]},"ts":"1731476503302"} 2024-11-13T05:41:43,303 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T05:41:43,304 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:41:43,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-13T05:41:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-13T05:41:44,289 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:44,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23792410, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:44,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:44,295 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:44,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:44,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49e88072, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:44,297 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:44,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38862da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,298 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:44,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:44,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:44,301 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:41:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,37913,1731476379290 2024-11-13T05:41:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28ac5b1a 2024-11-13T05:41:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:41:44,303 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:41:44,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:44,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:44,305 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@806d9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:44,307 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfb3b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:44,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:44,309 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:44,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:44,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@445cb108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:44,310 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b18cec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:44,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:44,312 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76a55e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:44,313 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f553a78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:44,315 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:44,315 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c47188c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:44,316 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:44,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f898eec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:44,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:44,318 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:44,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:44,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:44,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fd7ecfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:44,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:44,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:44,320 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ee2d81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:44,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:44,322 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@419b1f80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:44,324 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd35db2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:44,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:44,326 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:44,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:44,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:44,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a5b0ddf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:44,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:44,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:44,328 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cd0e5dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:44,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:44,329 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ac5b152, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:44,331 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d074bc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:44,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:44,332 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f28639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:44,334 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a39cb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:41:44,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:41:44,335 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:41:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:41:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:41:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b17baf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:41:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:41:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:41:44,336 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:41:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30967ed1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:41:44,337 DEBUG [Thread-2020 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:44,337 DEBUG [Thread-2027 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:44,338 DEBUG [Thread-2035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:44,339 DEBUG [Thread-2029 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:44,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:44,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:44,340 DEBUG [Thread-2024 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:44,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:44,340 DEBUG [Thread-2029 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:44,340 DEBUG [Thread-2029 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:44,340 DEBUG [Thread-2018 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:44,340 DEBUG [Thread-2031 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:44,341 DEBUG [Thread-2031 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:44,341 DEBUG [Thread-2018 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:41:44,341 DEBUG [Thread-2018 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:44,341 DEBUG [Thread-2031 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:44,342 DEBUG [Thread-2022 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:44,342 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:41:44,342 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:41:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:41:44,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:41:44,344 DEBUG [Thread-2016 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:44,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:44,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:44,346 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:44,347 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:44,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:41:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:41:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-13T05:41:44,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:41:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T05:41:44,350 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:41:44,351 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:44,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:44,355 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:44,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T05:41:44,367 DEBUG [Thread-2033 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:41:44,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:41:44,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:41:44,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:41:44,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2] 2024-11-13T05:41:44,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/db2be2f3a5e44dc98a8b1dcc9c3facbd is 50, key is test_row_0/A:col10/1731476504354/Put/seqid=0 2024-11-13T05:41:44,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742275_1451 (size=16681) 2024-11-13T05:41:44,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476564429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476564430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476564431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476564438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476564438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T05:41:44,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T05:41:44,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:44,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:44,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:44,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:44,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:44,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476564546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476564546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476564546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476564546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476564546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T05:41:44,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:44,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:44,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:44,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:44,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T05:41:44,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476564756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476564756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476564757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476564757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476564757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:44,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/db2be2f3a5e44dc98a8b1dcc9c3facbd 2024-11-13T05:41:44,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/20c0654a214c4de3a8105283c1642e65 is 50, key is test_row_0/B:col10/1731476504354/Put/seqid=0 2024-11-13T05:41:44,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742276_1452 (size=12001) 2024-11-13T05:41:44,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T05:41:44,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:44,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:44,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:44,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:44,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:44,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T05:41:44,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:44,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:44,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:44,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:44,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:44,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T05:41:45,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:45,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476565066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476565066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476565067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:45,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476565067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:45,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:45,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476565067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:45,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T05:41:45,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:45,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:45,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:45,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:45,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:45,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/20c0654a214c4de3a8105283c1642e65 2024-11-13T05:41:45,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/e7f7d7d8f6e54557b64169f9861f4b47 is 50, key is test_row_0/C:col10/1731476504354/Put/seqid=0 2024-11-13T05:41:45,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742277_1453 (size=12001) 2024-11-13T05:41:45,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T05:41:45,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:45,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:45,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:45,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:45,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:45,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T05:41:45,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:45,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:45,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:45,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:45,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T05:41:45,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T05:41:45,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:45,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:45,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:45,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:45,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:45,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476565577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:45,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476565577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:45,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476565577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476565577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476565577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:45,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/e7f7d7d8f6e54557b64169f9861f4b47 2024-11-13T05:41:45,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/db2be2f3a5e44dc98a8b1dcc9c3facbd as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/db2be2f3a5e44dc98a8b1dcc9c3facbd 2024-11-13T05:41:45,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/db2be2f3a5e44dc98a8b1dcc9c3facbd, entries=250, sequenceid=16, filesize=16.3 K 2024-11-13T05:41:45,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/20c0654a214c4de3a8105283c1642e65 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/20c0654a214c4de3a8105283c1642e65 2024-11-13T05:41:45,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/20c0654a214c4de3a8105283c1642e65, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T05:41:45,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/e7f7d7d8f6e54557b64169f9861f4b47 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e7f7d7d8f6e54557b64169f9861f4b47 2024-11-13T05:41:45,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e7f7d7d8f6e54557b64169f9861f4b47, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T05:41:45,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 666f8f0c32525e1ac64394208a6558b0 in 1293ms, sequenceid=16, compaction requested=false 2024-11-13T05:41:45,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:45,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T05:41:45,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:45,728 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T05:41:45,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/ed48e6a03ab54060a689beb98f11a4d6 is 50, key is test_row_0/A:col10/1731476504436/Put/seqid=0 2024-11-13T05:41:45,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742278_1454 (size=12001) 2024-11-13T05:41:45,740 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/ed48e6a03ab54060a689beb98f11a4d6 2024-11-13T05:41:45,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/c233d1d5e7c446378146abd9740416ea is 50, key is test_row_0/B:col10/1731476504436/Put/seqid=0 2024-11-13T05:41:45,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742279_1455 (size=12001) 2024-11-13T05:41:46,151 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/c233d1d5e7c446378146abd9740416ea 2024-11-13T05:41:46,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/e9a84bed330f41a29bc5f9038d495126 is 50, key is test_row_0/C:col10/1731476504436/Put/seqid=0 2024-11-13T05:41:46,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742280_1456 (size=12001) 2024-11-13T05:41:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T05:41:46,572 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/e9a84bed330f41a29bc5f9038d495126 2024-11-13T05:41:46,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/ed48e6a03ab54060a689beb98f11a4d6 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/ed48e6a03ab54060a689beb98f11a4d6 2024-11-13T05:41:46,578 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/ed48e6a03ab54060a689beb98f11a4d6, entries=150, sequenceid=38, filesize=11.7 K 2024-11-13T05:41:46,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/c233d1d5e7c446378146abd9740416ea as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/c233d1d5e7c446378146abd9740416ea 2024-11-13T05:41:46,582 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/c233d1d5e7c446378146abd9740416ea, entries=150, sequenceid=38, filesize=11.7 K 2024-11-13T05:41:46,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/e9a84bed330f41a29bc5f9038d495126 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e9a84bed330f41a29bc5f9038d495126 2024-11-13T05:41:46,585 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e9a84bed330f41a29bc5f9038d495126, entries=150, sequenceid=38, filesize=11.7 K 2024-11-13T05:41:46,586 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 666f8f0c32525e1ac64394208a6558b0 in 859ms, sequenceid=38, compaction requested=false 2024-11-13T05:41:46,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:46,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:46,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-13T05:41:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-13T05:41:46,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-13T05:41:46,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2370 sec 2024-11-13T05:41:46,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.2430 sec 2024-11-13T05:41:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:46,599 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:46,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:41:46,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/acd5d450985e4c60bef8f89260f63944 is 50, key is test_row_0/A:col10/1731476506598/Put/seqid=0 2024-11-13T05:41:46,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742281_1457 (size=19021) 2024-11-13T05:41:46,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476566640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476566641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:46,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476566644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:46,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476566645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476566653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:46,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476566756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476566756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476566756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:46,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476566767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:46,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476566767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:46,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476566966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476566966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476566966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:46,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476566976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:46,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:46,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476566986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:47,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/acd5d450985e4c60bef8f89260f63944 2024-11-13T05:41:47,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/4d958dd41a4d44ea986e171add0797f5 is 50, key is test_row_0/B:col10/1731476506598/Put/seqid=0 2024-11-13T05:41:47,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742282_1458 (size=12001) 2024-11-13T05:41:47,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476567276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476567276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:47,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476567286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:47,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:47,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476567287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:47,302 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:41:47,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476567296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:47,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/4d958dd41a4d44ea986e171add0797f5 2024-11-13T05:41:47,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/a000ca8034934c19aa770703acf2ec43 is 50, key is test_row_0/C:col10/1731476506598/Put/seqid=0 2024-11-13T05:41:47,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742283_1459 (size=12001) 2024-11-13T05:41:47,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/a000ca8034934c19aa770703acf2ec43 2024-11-13T05:41:47,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/acd5d450985e4c60bef8f89260f63944 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/acd5d450985e4c60bef8f89260f63944 2024-11-13T05:41:47,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/acd5d450985e4c60bef8f89260f63944, entries=300, sequenceid=49, filesize=18.6 K 2024-11-13T05:41:47,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/4d958dd41a4d44ea986e171add0797f5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/4d958dd41a4d44ea986e171add0797f5 2024-11-13T05:41:47,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/4d958dd41a4d44ea986e171add0797f5, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T05:41:47,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/a000ca8034934c19aa770703acf2ec43 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/a000ca8034934c19aa770703acf2ec43 2024-11-13T05:41:47,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/a000ca8034934c19aa770703acf2ec43, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T05:41:47,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 666f8f0c32525e1ac64394208a6558b0 in 851ms, sequenceid=49, compaction requested=true 2024-11-13T05:41:47,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:47,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:47,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:47,450 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:47,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:47,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:47,450 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:47,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:47,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:47,451 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:47,451 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/A is initiating minor compaction (all files) 2024-11-13T05:41:47,451 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/A in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:47,451 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/db2be2f3a5e44dc98a8b1dcc9c3facbd, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/ed48e6a03ab54060a689beb98f11a4d6, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/acd5d450985e4c60bef8f89260f63944] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=46.6 K 2024-11-13T05:41:47,452 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:47,452 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting db2be2f3a5e44dc98a8b1dcc9c3facbd, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731476504348 2024-11-13T05:41:47,452 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/B is initiating minor compaction (all files) 2024-11-13T05:41:47,452 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/B in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:47,452 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/20c0654a214c4de3a8105283c1642e65, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/c233d1d5e7c446378146abd9740416ea, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/4d958dd41a4d44ea986e171add0797f5] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=35.2 K 2024-11-13T05:41:47,452 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed48e6a03ab54060a689beb98f11a4d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731476504428 2024-11-13T05:41:47,452 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 20c0654a214c4de3a8105283c1642e65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731476504348 2024-11-13T05:41:47,452 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting acd5d450985e4c60bef8f89260f63944, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731476506588 2024-11-13T05:41:47,453 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting c233d1d5e7c446378146abd9740416ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731476504428 2024-11-13T05:41:47,453 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d958dd41a4d44ea986e171add0797f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731476506588 2024-11-13T05:41:47,457 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#B#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:47,458 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/7f927d8b45c047ad964c7400e9d8840e is 50, key is test_row_0/B:col10/1731476506598/Put/seqid=0 2024-11-13T05:41:47,458 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#A#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:47,459 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/f07ccc20d8f24e88af6c1a6ca73e60c5 is 50, key is test_row_0/A:col10/1731476506598/Put/seqid=0 2024-11-13T05:41:47,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742284_1460 (size=12104) 2024-11-13T05:41:47,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742285_1461 (size=12104) 2024-11-13T05:41:47,474 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/7f927d8b45c047ad964c7400e9d8840e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/7f927d8b45c047ad964c7400e9d8840e 2024-11-13T05:41:47,484 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/B of 666f8f0c32525e1ac64394208a6558b0 into 7f927d8b45c047ad964c7400e9d8840e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:47,484 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:47,484 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/B, priority=13, startTime=1731476507450; duration=0sec 2024-11-13T05:41:47,484 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:47,484 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:B 2024-11-13T05:41:47,484 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:47,485 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:47,485 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/C is initiating minor compaction (all files) 2024-11-13T05:41:47,485 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/C in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:47,485 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e7f7d7d8f6e54557b64169f9861f4b47, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e9a84bed330f41a29bc5f9038d495126, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/a000ca8034934c19aa770703acf2ec43] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=35.2 K 2024-11-13T05:41:47,485 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting e7f7d7d8f6e54557b64169f9861f4b47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731476504348 2024-11-13T05:41:47,486 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting e9a84bed330f41a29bc5f9038d495126, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731476504428 2024-11-13T05:41:47,486 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting a000ca8034934c19aa770703acf2ec43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731476506588 2024-11-13T05:41:47,493 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#C#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:47,494 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/956caf6f5e76427aa349d98e699919dc is 50, key is test_row_0/C:col10/1731476506598/Put/seqid=0 2024-11-13T05:41:47,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742286_1462 (size=12104) 2024-11-13T05:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:47,793 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:47,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T05:41:47,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/adecc36bbf4346928bbdc8e48e8b60ff is 50, key is test_row_0/A:col10/1731476506640/Put/seqid=0 2024-11-13T05:41:47,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742287_1463 (size=14341) 2024-11-13T05:41:47,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476567814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:47,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476567814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:47,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476567815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:47,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476567816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:47,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476567816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:47,876 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/f07ccc20d8f24e88af6c1a6ca73e60c5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f07ccc20d8f24e88af6c1a6ca73e60c5 2024-11-13T05:41:47,879 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/A of 666f8f0c32525e1ac64394208a6558b0 into f07ccc20d8f24e88af6c1a6ca73e60c5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:47,879 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:47,879 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/A, priority=13, startTime=1731476507450; duration=0sec 2024-11-13T05:41:47,879 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:47,879 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:A 2024-11-13T05:41:47,908 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/956caf6f5e76427aa349d98e699919dc as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/956caf6f5e76427aa349d98e699919dc 2024-11-13T05:41:47,913 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/C of 666f8f0c32525e1ac64394208a6558b0 into 956caf6f5e76427aa349d98e699919dc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:47,913 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:47,913 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/C, priority=13, startTime=1731476507450; duration=0sec 2024-11-13T05:41:47,913 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:47,913 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:C 2024-11-13T05:41:47,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476567926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:47,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476567926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:47,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476567936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476567937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:48,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476568136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476568136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:48,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476568146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:48,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:48,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476568146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:48,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:48,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/adecc36bbf4346928bbdc8e48e8b60ff 2024-11-13T05:41:48,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/cc79574c228f41a2b9060a604bc89389 is 50, key is test_row_0/B:col10/1731476506640/Put/seqid=0 2024-11-13T05:41:48,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742288_1464 (size=12001) 2024-11-13T05:41:48,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476568446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476568446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:48,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476568456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:48,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476568457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T05:41:48,506 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:48,507 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-13T05:41:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T05:41:48,508 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:48,509 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:48,509 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:48,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/cc79574c228f41a2b9060a604bc89389 2024-11-13T05:41:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T05:41:48,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/0f645041c8344a699c34a6cd30c04296 is 50, key is test_row_0/C:col10/1731476506640/Put/seqid=0 2024-11-13T05:41:48,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742289_1465 (size=12001) 2024-11-13T05:41:48,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T05:41:48,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:48,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:48,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:48,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:48,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:48,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T05:41:48,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:48,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:48,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:48,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:48,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T05:41:48,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:48,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476568836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:48,844 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in 666f8f0c32525e1ac64394208a6558b0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:48,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476568956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476568956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:48,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T05:41:48,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:48,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:48,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:48,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:48,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:48,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476568966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:48,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476568976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:48,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:48,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:49,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/0f645041c8344a699c34a6cd30c04296 2024-11-13T05:41:49,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/adecc36bbf4346928bbdc8e48e8b60ff as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/adecc36bbf4346928bbdc8e48e8b60ff 2024-11-13T05:41:49,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/adecc36bbf4346928bbdc8e48e8b60ff, entries=200, sequenceid=76, filesize=14.0 K 2024-11-13T05:41:49,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/cc79574c228f41a2b9060a604bc89389 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/cc79574c228f41a2b9060a604bc89389 2024-11-13T05:41:49,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/cc79574c228f41a2b9060a604bc89389, entries=150, sequenceid=76, filesize=11.7 K 2024-11-13T05:41:49,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/0f645041c8344a699c34a6cd30c04296 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/0f645041c8344a699c34a6cd30c04296 2024-11-13T05:41:49,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/0f645041c8344a699c34a6cd30c04296, entries=150, sequenceid=76, filesize=11.7 K 2024-11-13T05:41:49,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 666f8f0c32525e1ac64394208a6558b0 in 1241ms, sequenceid=76, compaction requested=false 2024-11-13T05:41:49,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:49,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T05:41:49,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:49,119 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-13T05:41:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/a6eb37b3e79749ed8e6626ab70a23c1a is 50, key is test_row_0/A:col10/1731476507815/Put/seqid=0 2024-11-13T05:41:49,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742290_1466 (size=12001) 2024-11-13T05:41:49,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T05:41:49,526 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/a6eb37b3e79749ed8e6626ab70a23c1a 2024-11-13T05:41:49,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/7c9c3d991cd34a92911ee84799ee715e is 50, key is test_row_0/B:col10/1731476507815/Put/seqid=0 2024-11-13T05:41:49,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742291_1467 (size=12001) 2024-11-13T05:41:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T05:41:49,936 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/7c9c3d991cd34a92911ee84799ee715e 2024-11-13T05:41:49,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/bcd1e0eb8af74d69936ceb2c2f5d8b5e is 50, key is test_row_0/C:col10/1731476507815/Put/seqid=0 2024-11-13T05:41:49,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742292_1468 (size=12001) 2024-11-13T05:41:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:49,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:50,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476570032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476570033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476570034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476570034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476570146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476570146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476570146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476570147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,347 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/bcd1e0eb8af74d69936ceb2c2f5d8b5e 2024-11-13T05:41:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/a6eb37b3e79749ed8e6626ab70a23c1a as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/a6eb37b3e79749ed8e6626ab70a23c1a 2024-11-13T05:41:50,353 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/a6eb37b3e79749ed8e6626ab70a23c1a, entries=150, sequenceid=88, filesize=11.7 K 2024-11-13T05:41:50,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/7c9c3d991cd34a92911ee84799ee715e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/7c9c3d991cd34a92911ee84799ee715e 2024-11-13T05:41:50,356 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/7c9c3d991cd34a92911ee84799ee715e, entries=150, sequenceid=88, filesize=11.7 K 2024-11-13T05:41:50,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/bcd1e0eb8af74d69936ceb2c2f5d8b5e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/bcd1e0eb8af74d69936ceb2c2f5d8b5e 2024-11-13T05:41:50,360 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/bcd1e0eb8af74d69936ceb2c2f5d8b5e, entries=150, sequenceid=88, filesize=11.7 K 2024-11-13T05:41:50,360 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 666f8f0c32525e1ac64394208a6558b0 in 1241ms, sequenceid=88, compaction requested=true 2024-11-13T05:41:50,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:50,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:50,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-13T05:41:50,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-13T05:41:50,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-13T05:41:50,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8530 sec 2024-11-13T05:41:50,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.8560 sec 2024-11-13T05:41:50,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:50,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:50,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-13T05:41:50,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/acb7255f76e64bf69ae280b22c51b1c3 is 50, key is test_row_0/A:col10/1731476510032/Put/seqid=0 2024-11-13T05:41:50,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742293_1469 (size=14341) 2024-11-13T05:41:50,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476570377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476570378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476570378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476570379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476570487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476570496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476570496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476570496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T05:41:50,657 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:50,657 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-13T05:41:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T05:41:50,658 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:50,659 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:50,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:50,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476570706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476570706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476570716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476570716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T05:41:50,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/acb7255f76e64bf69ae280b22c51b1c3 2024-11-13T05:41:50,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/d9a867e2241a4032924c0bfe7f6e864e is 50, key is test_row_0/B:col10/1731476510032/Put/seqid=0 2024-11-13T05:41:50,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742294_1470 (size=12001) 2024-11-13T05:41:50,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:41:50,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:50,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:50,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:50,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:50,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:50,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476570857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:50,861 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in 666f8f0c32525e1ac64394208a6558b0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:50,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:41:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:50,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T05:41:51,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476571016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:51,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:51,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476571026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:51,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:51,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476571026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:51,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:51,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:51,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476571036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:51,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:51,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:41:51,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:51,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:51,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:51,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:51,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:51,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/d9a867e2241a4032924c0bfe7f6e864e 2024-11-13T05:41:51,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/74ae11e787f14db98527bbc19932c1bd is 50, key is test_row_0/C:col10/1731476510032/Put/seqid=0 2024-11-13T05:41:51,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742295_1471 (size=12001) 2024-11-13T05:41:51,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:41:51,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:51,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:51,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:51,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:51,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T05:41:51,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:41:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:51,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:51,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476571526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:51,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476571536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:51,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:51,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476571536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:51,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476571546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:51,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:51,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:51,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:41:51,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:51,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:51,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:51,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:51,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:51,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/74ae11e787f14db98527bbc19932c1bd 2024-11-13T05:41:51,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/acb7255f76e64bf69ae280b22c51b1c3 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/acb7255f76e64bf69ae280b22c51b1c3 2024-11-13T05:41:51,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/acb7255f76e64bf69ae280b22c51b1c3, entries=200, sequenceid=115, filesize=14.0 K 2024-11-13T05:41:51,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/d9a867e2241a4032924c0bfe7f6e864e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/d9a867e2241a4032924c0bfe7f6e864e 2024-11-13T05:41:51,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/d9a867e2241a4032924c0bfe7f6e864e, entries=150, sequenceid=115, filesize=11.7 K 2024-11-13T05:41:51,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/74ae11e787f14db98527bbc19932c1bd as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/74ae11e787f14db98527bbc19932c1bd 2024-11-13T05:41:51,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/74ae11e787f14db98527bbc19932c1bd, entries=150, sequenceid=115, filesize=11.7 K 2024-11-13T05:41:51,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 666f8f0c32525e1ac64394208a6558b0 in 1243ms, sequenceid=115, compaction requested=true 2024-11-13T05:41:51,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:51,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:51,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:51,607 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:41:51,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:51,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:51,607 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:41:51,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:51,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:51,608 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:41:51,608 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:41:51,608 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/A is initiating minor compaction (all files) 2024-11-13T05:41:51,608 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/B is initiating minor compaction (all files) 2024-11-13T05:41:51,609 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/B in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:51,609 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/A in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:51,609 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/7f927d8b45c047ad964c7400e9d8840e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/cc79574c228f41a2b9060a604bc89389, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/7c9c3d991cd34a92911ee84799ee715e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/d9a867e2241a4032924c0bfe7f6e864e] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=47.0 K 2024-11-13T05:41:51,609 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f07ccc20d8f24e88af6c1a6ca73e60c5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/adecc36bbf4346928bbdc8e48e8b60ff, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/a6eb37b3e79749ed8e6626ab70a23c1a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/acb7255f76e64bf69ae280b22c51b1c3] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=51.5 K 2024-11-13T05:41:51,609 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f927d8b45c047ad964c7400e9d8840e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731476506588 2024-11-13T05:41:51,609 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting f07ccc20d8f24e88af6c1a6ca73e60c5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731476506588 2024-11-13T05:41:51,609 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting cc79574c228f41a2b9060a604bc89389, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731476506640 2024-11-13T05:41:51,609 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting adecc36bbf4346928bbdc8e48e8b60ff, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731476506640 2024-11-13T05:41:51,609 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c9c3d991cd34a92911ee84799ee715e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731476507803 2024-11-13T05:41:51,609 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6eb37b3e79749ed8e6626ab70a23c1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731476507803 2024-11-13T05:41:51,609 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a867e2241a4032924c0bfe7f6e864e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731476510032 2024-11-13T05:41:51,609 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting acb7255f76e64bf69ae280b22c51b1c3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731476510030 2024-11-13T05:41:51,615 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:51,615 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/2d4c9032ff724a72825056c9a2b6a468 is 50, key is test_row_0/B:col10/1731476510032/Put/seqid=0 2024-11-13T05:41:51,616 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#A#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:51,616 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/12f51dcc1f9d44c09774b30ca5096fd3 is 50, key is test_row_0/A:col10/1731476510032/Put/seqid=0 2024-11-13T05:41:51,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742296_1472 (size=12241) 2024-11-13T05:41:51,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742297_1473 (size=12241) 2024-11-13T05:41:51,623 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/2d4c9032ff724a72825056c9a2b6a468 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/2d4c9032ff724a72825056c9a2b6a468 2024-11-13T05:41:51,628 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/B of 666f8f0c32525e1ac64394208a6558b0 into 2d4c9032ff724a72825056c9a2b6a468(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:51,628 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:51,628 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/B, priority=12, startTime=1731476511607; duration=0sec 2024-11-13T05:41:51,628 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:51,628 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:B 2024-11-13T05:41:51,628 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:41:51,629 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:41:51,629 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/C is initiating minor compaction (all files) 2024-11-13T05:41:51,629 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/C in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:51,629 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/956caf6f5e76427aa349d98e699919dc, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/0f645041c8344a699c34a6cd30c04296, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/bcd1e0eb8af74d69936ceb2c2f5d8b5e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/74ae11e787f14db98527bbc19932c1bd] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=47.0 K 2024-11-13T05:41:51,629 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 956caf6f5e76427aa349d98e699919dc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731476506588 2024-11-13T05:41:51,630 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f645041c8344a699c34a6cd30c04296, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731476506640 2024-11-13T05:41:51,630 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting bcd1e0eb8af74d69936ceb2c2f5d8b5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731476507803 2024-11-13T05:41:51,630 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 74ae11e787f14db98527bbc19932c1bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731476510032 2024-11-13T05:41:51,633 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/12f51dcc1f9d44c09774b30ca5096fd3 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/12f51dcc1f9d44c09774b30ca5096fd3 2024-11-13T05:41:51,638 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/A of 666f8f0c32525e1ac64394208a6558b0 into 12f51dcc1f9d44c09774b30ca5096fd3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:51,638 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:51,638 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/A, priority=12, startTime=1731476511607; duration=0sec 2024-11-13T05:41:51,638 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:51,638 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:A 2024-11-13T05:41:51,639 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#C#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:51,640 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/0ec1c4ee1a29454fbe2b963ce84be3ae is 50, key is test_row_0/C:col10/1731476510032/Put/seqid=0 2024-11-13T05:41:51,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742298_1474 (size=12241) 2024-11-13T05:41:51,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:41:51,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:51,726 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-13T05:41:51,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/05fdbe9c94bb4e9bbc52a35a33245925 is 50, key is test_row_0/A:col10/1731476510376/Put/seqid=0 2024-11-13T05:41:51,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742299_1475 (size=12001) 2024-11-13T05:41:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T05:41:52,051 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/0ec1c4ee1a29454fbe2b963ce84be3ae as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/0ec1c4ee1a29454fbe2b963ce84be3ae 2024-11-13T05:41:52,054 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/C of 666f8f0c32525e1ac64394208a6558b0 into 0ec1c4ee1a29454fbe2b963ce84be3ae(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:52,054 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:52,054 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/C, priority=12, startTime=1731476511607; duration=0sec 2024-11-13T05:41:52,054 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:52,055 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:C 2024-11-13T05:41:52,136 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/05fdbe9c94bb4e9bbc52a35a33245925 2024-11-13T05:41:52,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/400858ea25604d9389fbfa948ad8c582 is 50, key is test_row_0/B:col10/1731476510376/Put/seqid=0 2024-11-13T05:41:52,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742300_1476 (size=12001) 2024-11-13T05:41:52,147 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/400858ea25604d9389fbfa948ad8c582 2024-11-13T05:41:52,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/08035b27da8a4440bad2d8476f734fa8 is 50, key is test_row_0/C:col10/1731476510376/Put/seqid=0 2024-11-13T05:41:52,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742301_1477 (size=12001) 2024-11-13T05:41:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:52,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:52,556 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/08035b27da8a4440bad2d8476f734fa8 2024-11-13T05:41:52,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/05fdbe9c94bb4e9bbc52a35a33245925 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/05fdbe9c94bb4e9bbc52a35a33245925 2024-11-13T05:41:52,563 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/05fdbe9c94bb4e9bbc52a35a33245925, entries=150, sequenceid=127, filesize=11.7 K 2024-11-13T05:41:52,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/400858ea25604d9389fbfa948ad8c582 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/400858ea25604d9389fbfa948ad8c582 2024-11-13T05:41:52,567 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/400858ea25604d9389fbfa948ad8c582, entries=150, sequenceid=127, filesize=11.7 K 2024-11-13T05:41:52,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/08035b27da8a4440bad2d8476f734fa8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/08035b27da8a4440bad2d8476f734fa8 2024-11-13T05:41:52,571 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/08035b27da8a4440bad2d8476f734fa8, entries=150, sequenceid=127, filesize=11.7 K 2024-11-13T05:41:52,601 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=40.25 KB/41220 for 666f8f0c32525e1ac64394208a6558b0 in 875ms, sequenceid=127, compaction requested=false 2024-11-13T05:41:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:52,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:52,602 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:52,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:52,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-13T05:41:52,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T05:41:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-13T05:41:52,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-13T05:41:52,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9440 sec 2024-11-13T05:41:52,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.9490 sec 2024-11-13T05:41:52,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/87937115aead4bc9a2014cd89d19dabd is 50, key is test_row_0/A:col10/1731476512567/Put/seqid=0 2024-11-13T05:41:52,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742302_1478 (size=14541) 2024-11-13T05:41:52,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476572652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476572657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:52,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476572658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:52,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476572666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:52,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:52,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476572776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476572776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476572776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:52,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476572786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:52,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T05:41:52,807 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:52,807 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-13T05:41:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T05:41:52,809 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:52,809 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:52,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T05:41:52,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T05:41:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:52,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:52,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476572986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:52,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476572986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:53,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476572996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:53,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476573006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:53,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:53,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/87937115aead4bc9a2014cd89d19dabd 2024-11-13T05:41:53,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/a402f182caec4b15a94d75972f0bd7b1 is 50, key is test_row_0/B:col10/1731476512567/Put/seqid=0 2024-11-13T05:41:53,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742303_1479 (size=12151) 2024-11-13T05:41:53,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T05:41:53,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:53,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:53,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:53,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:53,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T05:41:53,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T05:41:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:53,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:53,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476573297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:53,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:53,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476573306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:53,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476573316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476573316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:53,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T05:41:53,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:53,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:53,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:53,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:53,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:53,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/a402f182caec4b15a94d75972f0bd7b1 2024-11-13T05:41:53,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/8e24181d40a8428fa45c74aaabf1b193 is 50, key is test_row_0/C:col10/1731476512567/Put/seqid=0 2024-11-13T05:41:53,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742304_1480 (size=12151) 2024-11-13T05:41:53,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/8e24181d40a8428fa45c74aaabf1b193 2024-11-13T05:41:53,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/87937115aead4bc9a2014cd89d19dabd as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/87937115aead4bc9a2014cd89d19dabd 2024-11-13T05:41:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T05:41:53,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/87937115aead4bc9a2014cd89d19dabd, entries=200, sequenceid=141, filesize=14.2 K 2024-11-13T05:41:53,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/a402f182caec4b15a94d75972f0bd7b1 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/a402f182caec4b15a94d75972f0bd7b1 2024-11-13T05:41:53,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/a402f182caec4b15a94d75972f0bd7b1, entries=150, sequenceid=141, filesize=11.9 K 2024-11-13T05:41:53,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/8e24181d40a8428fa45c74aaabf1b193 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/8e24181d40a8428fa45c74aaabf1b193 2024-11-13T05:41:53,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/8e24181d40a8428fa45c74aaabf1b193, entries=150, sequenceid=141, filesize=11.9 K 2024-11-13T05:41:53,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 666f8f0c32525e1ac64394208a6558b0 in 844ms, sequenceid=141, compaction requested=true 2024-11-13T05:41:53,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:53,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:53,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:53,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:53,447 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:53,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:53,447 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:53,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:53,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:53,447 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:53,447 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:53,447 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/A is initiating minor compaction (all files) 2024-11-13T05:41:53,447 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/B is initiating minor compaction (all files) 2024-11-13T05:41:53,447 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/A in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:53,447 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/B in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:53,447 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/12f51dcc1f9d44c09774b30ca5096fd3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/05fdbe9c94bb4e9bbc52a35a33245925, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/87937115aead4bc9a2014cd89d19dabd] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=37.9 K 2024-11-13T05:41:53,447 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/2d4c9032ff724a72825056c9a2b6a468, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/400858ea25604d9389fbfa948ad8c582, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/a402f182caec4b15a94d75972f0bd7b1] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=35.5 K 2024-11-13T05:41:53,448 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12f51dcc1f9d44c09774b30ca5096fd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731476510032 2024-11-13T05:41:53,448 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d4c9032ff724a72825056c9a2b6a468, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731476510032 2024-11-13T05:41:53,448 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05fdbe9c94bb4e9bbc52a35a33245925, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731476510375 2024-11-13T05:41:53,448 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 400858ea25604d9389fbfa948ad8c582, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731476510375 2024-11-13T05:41:53,448 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting a402f182caec4b15a94d75972f0bd7b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731476512556 2024-11-13T05:41:53,448 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87937115aead4bc9a2014cd89d19dabd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731476512556 2024-11-13T05:41:53,456 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#B#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:53,456 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/63bb438a9dfd40e888ff13b789004e56 is 50, key is test_row_0/B:col10/1731476512567/Put/seqid=0 2024-11-13T05:41:53,457 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#A#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:53,457 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/f0acad97281e4b01a7f8267d62b94bbb is 50, key is test_row_0/A:col10/1731476512567/Put/seqid=0 2024-11-13T05:41:53,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742306_1482 (size=12493) 2024-11-13T05:41:53,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742305_1481 (size=12493) 2024-11-13T05:41:53,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-13T05:41:53,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:53,572 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T05:41:53,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/7260c098e707400fb465992903fbbd98 is 50, key is test_row_0/A:col10/1731476512656/Put/seqid=0 2024-11-13T05:41:53,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742307_1483 (size=12151) 2024-11-13T05:41:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:53,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:53,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476573836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:53,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476573838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:53,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476573842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:53,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476573849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:53,875 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/f0acad97281e4b01a7f8267d62b94bbb as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f0acad97281e4b01a7f8267d62b94bbb 2024-11-13T05:41:53,875 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/63bb438a9dfd40e888ff13b789004e56 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/63bb438a9dfd40e888ff13b789004e56 2024-11-13T05:41:53,879 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/B of 666f8f0c32525e1ac64394208a6558b0 into 63bb438a9dfd40e888ff13b789004e56(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:53,879 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:53,879 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/B, priority=13, startTime=1731476513446; duration=0sec 2024-11-13T05:41:53,879 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:53,879 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:B 2024-11-13T05:41:53,879 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:53,879 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/A of 666f8f0c32525e1ac64394208a6558b0 into f0acad97281e4b01a7f8267d62b94bbb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:53,879 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:53,879 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/A, priority=13, startTime=1731476513446; duration=0sec 2024-11-13T05:41:53,879 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:53,879 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:A 2024-11-13T05:41:53,880 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:53,880 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/C is initiating minor compaction (all files) 2024-11-13T05:41:53,880 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/C in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:53,880 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/0ec1c4ee1a29454fbe2b963ce84be3ae, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/08035b27da8a4440bad2d8476f734fa8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/8e24181d40a8428fa45c74aaabf1b193] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=35.5 K 2024-11-13T05:41:53,880 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ec1c4ee1a29454fbe2b963ce84be3ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731476510032 2024-11-13T05:41:53,880 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 08035b27da8a4440bad2d8476f734fa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731476510375 2024-11-13T05:41:53,880 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e24181d40a8428fa45c74aaabf1b193, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731476512556 2024-11-13T05:41:53,886 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#C#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:53,886 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/66daab82e04542edb5cffc98015e8500 is 50, key is test_row_0/C:col10/1731476512567/Put/seqid=0 2024-11-13T05:41:53,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742308_1484 (size=12493) 2024-11-13T05:41:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T05:41:53,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476573956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:53,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476573956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:53,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476573966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:53,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:53,981 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/7260c098e707400fb465992903fbbd98 2024-11-13T05:41:53,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/24cc9bc17baa4cbaa5511e3a2f835ae9 is 50, key is test_row_0/B:col10/1731476512656/Put/seqid=0 2024-11-13T05:41:53,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742309_1485 (size=12151) 2024-11-13T05:41:54,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476574176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:54,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476574176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476574176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:54,292 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/66daab82e04542edb5cffc98015e8500 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/66daab82e04542edb5cffc98015e8500 2024-11-13T05:41:54,296 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/C of 666f8f0c32525e1ac64394208a6558b0 into 66daab82e04542edb5cffc98015e8500(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:54,296 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:54,296 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/C, priority=13, startTime=1731476513447; duration=0sec 2024-11-13T05:41:54,296 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:54,296 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:C 2024-11-13T05:41:54,390 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/24cc9bc17baa4cbaa5511e3a2f835ae9 2024-11-13T05:41:54,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/c2ba2d77d60b40748815b8506720dbc9 is 50, key is test_row_0/C:col10/1731476512656/Put/seqid=0 2024-11-13T05:41:54,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742310_1486 (size=12151) 2024-11-13T05:41:54,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476574486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:54,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:54,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476574486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:54,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:54,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476574487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:54,800 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/c2ba2d77d60b40748815b8506720dbc9 2024-11-13T05:41:54,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/7260c098e707400fb465992903fbbd98 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/7260c098e707400fb465992903fbbd98 2024-11-13T05:41:54,841 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/7260c098e707400fb465992903fbbd98, entries=150, sequenceid=164, filesize=11.9 K 2024-11-13T05:41:54,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-13T05:41:54,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/24cc9bc17baa4cbaa5511e3a2f835ae9 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/24cc9bc17baa4cbaa5511e3a2f835ae9 2024-11-13T05:41:54,844 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/24cc9bc17baa4cbaa5511e3a2f835ae9, entries=150, sequenceid=164, filesize=11.9 K 2024-11-13T05:41:54,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/c2ba2d77d60b40748815b8506720dbc9 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/c2ba2d77d60b40748815b8506720dbc9 2024-11-13T05:41:54,847 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/c2ba2d77d60b40748815b8506720dbc9, entries=150, sequenceid=164, filesize=11.9 K 2024-11-13T05:41:54,848 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 666f8f0c32525e1ac64394208a6558b0 in 1276ms, sequenceid=164, compaction requested=false 2024-11-13T05:41:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-13T05:41:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-13T05:41:54,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-13T05:41:54,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0390 sec 2024-11-13T05:41:54,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.0440 sec 2024-11-13T05:41:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:54,863 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:54,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T05:41:54,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/c82c0bfe2620430ca3ff8ad46794ae70 is 50, key is test_row_0/A:col10/1731476513832/Put/seqid=0 2024-11-13T05:41:54,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742311_1487 (size=14541) 2024-11-13T05:41:54,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476574936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:54,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:54,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476574943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T05:41:54,957 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:54,957 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-13T05:41:54,958 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T05:41:54,959 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:54,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:55,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476574996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:55,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:55,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476575007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:55,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476575007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:55,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476575046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:55,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476575056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T05:41:55,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:55,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:55,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:55,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:55,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476575256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:55,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:55,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:55,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:55,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:55,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/c82c0bfe2620430ca3ff8ad46794ae70 2024-11-13T05:41:55,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476575266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:55,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:55,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T05:41:55,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/56ab6fa3ebb54afc8b78832c46cb5907 is 50, key is test_row_0/B:col10/1731476513832/Put/seqid=0 2024-11-13T05:41:55,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742312_1488 (size=12151) 2024-11-13T05:41:55,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:55,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:55,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:55,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:55,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:55,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:55,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:55,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:55,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476575566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T05:41:55,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476575586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:55,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/56ab6fa3ebb54afc8b78832c46cb5907 2024-11-13T05:41:55,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/71f2e1ebea3d4d57b40e8f1832c33e27 is 50, key is test_row_0/C:col10/1731476513832/Put/seqid=0 2024-11-13T05:41:55,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742313_1489 (size=12151) 2024-11-13T05:41:55,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:55,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:55,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:55,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:55,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:55,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:55,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:55,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:55,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476576016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:56,024 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in 666f8f0c32525e1ac64394208a6558b0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:56,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:56,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:56,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476576027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:56,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476576027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:56,034 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in 666f8f0c32525e1ac64394208a6558b0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:56,034 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in 666f8f0c32525e1ac64394208a6558b0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:56,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476576086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:56,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/71f2e1ebea3d4d57b40e8f1832c33e27 2024-11-13T05:41:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T05:41:56,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/c82c0bfe2620430ca3ff8ad46794ae70 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/c82c0bfe2620430ca3ff8ad46794ae70 2024-11-13T05:41:56,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/c82c0bfe2620430ca3ff8ad46794ae70, entries=200, sequenceid=181, filesize=14.2 K 2024-11-13T05:41:56,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/56ab6fa3ebb54afc8b78832c46cb5907 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/56ab6fa3ebb54afc8b78832c46cb5907 2024-11-13T05:41:56,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/56ab6fa3ebb54afc8b78832c46cb5907, entries=150, sequenceid=181, filesize=11.9 K 2024-11-13T05:41:56,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/71f2e1ebea3d4d57b40e8f1832c33e27 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/71f2e1ebea3d4d57b40e8f1832c33e27 2024-11-13T05:41:56,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/71f2e1ebea3d4d57b40e8f1832c33e27, entries=150, sequenceid=181, filesize=11.9 K 2024-11-13T05:41:56,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 666f8f0c32525e1ac64394208a6558b0 in 1245ms, sequenceid=181, compaction requested=true 2024-11-13T05:41:56,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:56,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:56,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:56,108 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:56,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:56,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:56,108 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:56,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:56,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:56,109 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:56,109 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:56,109 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/A is initiating minor compaction (all files) 2024-11-13T05:41:56,109 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/B is initiating minor compaction (all files) 2024-11-13T05:41:56,109 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/A in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,109 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/B in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,109 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f0acad97281e4b01a7f8267d62b94bbb, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/7260c098e707400fb465992903fbbd98, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/c82c0bfe2620430ca3ff8ad46794ae70] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=38.3 K 2024-11-13T05:41:56,109 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/63bb438a9dfd40e888ff13b789004e56, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/24cc9bc17baa4cbaa5511e3a2f835ae9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/56ab6fa3ebb54afc8b78832c46cb5907] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=35.9 K 2024-11-13T05:41:56,109 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0acad97281e4b01a7f8267d62b94bbb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731476512556 2024-11-13T05:41:56,109 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 63bb438a9dfd40e888ff13b789004e56, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731476512556 2024-11-13T05:41:56,109 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7260c098e707400fb465992903fbbd98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731476512655 2024-11-13T05:41:56,109 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 24cc9bc17baa4cbaa5511e3a2f835ae9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731476512655 2024-11-13T05:41:56,110 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting c82c0bfe2620430ca3ff8ad46794ae70, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731476513832 2024-11-13T05:41:56,110 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ab6fa3ebb54afc8b78832c46cb5907, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731476513832 2024-11-13T05:41:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:56,114 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:56,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T05:41:56,116 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#A#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:56,117 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#B#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:56,117 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/c984d602c73144fab16dcfe967e8e9ca is 50, key is test_row_0/A:col10/1731476513832/Put/seqid=0 2024-11-13T05:41:56,117 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/3a8a965b14bc46839b97085a2152a5dd is 50, key is test_row_0/B:col10/1731476513832/Put/seqid=0 2024-11-13T05:41:56,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/f457b200ecab48458e1afdbb6003229b is 50, key is test_row_0/A:col10/1731476514934/Put/seqid=0 2024-11-13T05:41:56,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742314_1490 (size=12595) 2024-11-13T05:41:56,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742315_1491 (size=12595) 2024-11-13T05:41:56,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742316_1492 (size=14541) 2024-11-13T05:41:56,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:56,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:56,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476576204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:56,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476576316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:56,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:56,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:56,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:56,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:56,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476576526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:56,531 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/c984d602c73144fab16dcfe967e8e9ca as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/c984d602c73144fab16dcfe967e8e9ca 2024-11-13T05:41:56,535 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/A of 666f8f0c32525e1ac64394208a6558b0 into c984d602c73144fab16dcfe967e8e9ca(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:56,535 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:56,535 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/A, priority=13, startTime=1731476516108; duration=0sec 2024-11-13T05:41:56,535 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:56,535 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:A 2024-11-13T05:41:56,536 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:41:56,536 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:41:56,536 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/C is initiating minor compaction (all files) 2024-11-13T05:41:56,536 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/C in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,536 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/66daab82e04542edb5cffc98015e8500, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/c2ba2d77d60b40748815b8506720dbc9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/71f2e1ebea3d4d57b40e8f1832c33e27] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=35.9 K 2024-11-13T05:41:56,537 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66daab82e04542edb5cffc98015e8500, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731476512556 2024-11-13T05:41:56,537 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ba2d77d60b40748815b8506720dbc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731476512655 2024-11-13T05:41:56,537 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71f2e1ebea3d4d57b40e8f1832c33e27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731476513832 2024-11-13T05:41:56,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/f457b200ecab48458e1afdbb6003229b 2024-11-13T05:41:56,542 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/3a8a965b14bc46839b97085a2152a5dd as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/3a8a965b14bc46839b97085a2152a5dd 2024-11-13T05:41:56,546 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/B of 666f8f0c32525e1ac64394208a6558b0 into 3a8a965b14bc46839b97085a2152a5dd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:56,546 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:56,546 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/B, priority=13, startTime=1731476516108; duration=0sec 2024-11-13T05:41:56,546 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:56,546 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:B 2024-11-13T05:41:56,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/fd774faf06a04891a611d5cb8bbe058a is 50, key is test_row_0/B:col10/1731476514934/Put/seqid=0 2024-11-13T05:41:56,552 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#C#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:56,552 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/45aa94fa945a4a6eaef6e1579f455180 is 50, key is test_row_0/C:col10/1731476513832/Put/seqid=0 2024-11-13T05:41:56,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742317_1493 (size=12151) 2024-11-13T05:41:56,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742318_1494 (size=12595) 2024-11-13T05:41:56,567 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/45aa94fa945a4a6eaef6e1579f455180 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/45aa94fa945a4a6eaef6e1579f455180 2024-11-13T05:41:56,575 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/C of 666f8f0c32525e1ac64394208a6558b0 into 45aa94fa945a4a6eaef6e1579f455180(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:56,575 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:56,575 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/C, priority=13, startTime=1731476516108; duration=0sec 2024-11-13T05:41:56,575 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:56,575 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:C 2024-11-13T05:41:56,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:56,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:56,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:56,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:56,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476576836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:56,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:56,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:56,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:56,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:56,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/fd774faf06a04891a611d5cb8bbe058a 2024-11-13T05:41:56,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/e3e8a4c8ea214501b01efd097ef8c21e is 50, key is test_row_0/C:col10/1731476514934/Put/seqid=0 2024-11-13T05:41:56,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742319_1495 (size=12151) 2024-11-13T05:41:57,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:57,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:57,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:57,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:57,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:57,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T05:41:57,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476577106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:57,112 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in 666f8f0c32525e1ac64394208a6558b0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:57,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:57,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:57,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476577346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:57,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/e3e8a4c8ea214501b01efd097ef8c21e 2024-11-13T05:41:57,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/f457b200ecab48458e1afdbb6003229b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f457b200ecab48458e1afdbb6003229b 2024-11-13T05:41:57,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f457b200ecab48458e1afdbb6003229b, entries=200, sequenceid=204, filesize=14.2 K 2024-11-13T05:41:57,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/fd774faf06a04891a611d5cb8bbe058a as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/fd774faf06a04891a611d5cb8bbe058a 2024-11-13T05:41:57,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/fd774faf06a04891a611d5cb8bbe058a, entries=150, sequenceid=204, filesize=11.9 K 2024-11-13T05:41:57,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/e3e8a4c8ea214501b01efd097ef8c21e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e3e8a4c8ea214501b01efd097ef8c21e 2024-11-13T05:41:57,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e3e8a4c8ea214501b01efd097ef8c21e, entries=150, sequenceid=204, filesize=11.9 K 2024-11-13T05:41:57,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 666f8f0c32525e1ac64394208a6558b0 in 1266ms, sequenceid=204, compaction requested=false 2024-11-13T05:41:57,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:57,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-13T05:41:57,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:57,401 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T05:41:57,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/5becf385c2764aee94bd9288e614dad9 is 50, key is test_row_0/A:col10/1731476516184/Put/seqid=0 2024-11-13T05:41:57,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742320_1496 (size=12151) 2024-11-13T05:41:57,414 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/5becf385c2764aee94bd9288e614dad9 2024-11-13T05:41:57,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/98e3bce2cf364241838ddd5aa68e1cf8 is 50, key is test_row_0/B:col10/1731476516184/Put/seqid=0 2024-11-13T05:41:57,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742321_1497 (size=12151) 2024-11-13T05:41:57,424 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/98e3bce2cf364241838ddd5aa68e1cf8 2024-11-13T05:41:57,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/125d3cb3b11245b3b57530a6c2de73b3 is 50, key is test_row_0/C:col10/1731476516184/Put/seqid=0 2024-11-13T05:41:57,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742322_1498 (size=12151) 2024-11-13T05:41:57,835 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/125d3cb3b11245b3b57530a6c2de73b3 2024-11-13T05:41:57,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/5becf385c2764aee94bd9288e614dad9 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/5becf385c2764aee94bd9288e614dad9 2024-11-13T05:41:57,841 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/5becf385c2764aee94bd9288e614dad9, entries=150, sequenceid=220, filesize=11.9 K 2024-11-13T05:41:57,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/98e3bce2cf364241838ddd5aa68e1cf8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/98e3bce2cf364241838ddd5aa68e1cf8 2024-11-13T05:41:57,847 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/98e3bce2cf364241838ddd5aa68e1cf8, entries=150, sequenceid=220, filesize=11.9 K 2024-11-13T05:41:57,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/125d3cb3b11245b3b57530a6c2de73b3 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/125d3cb3b11245b3b57530a6c2de73b3 2024-11-13T05:41:57,850 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/125d3cb3b11245b3b57530a6c2de73b3, entries=150, sequenceid=220, filesize=11.9 K 2024-11-13T05:41:57,851 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 666f8f0c32525e1ac64394208a6558b0 in 451ms, sequenceid=220, compaction requested=true 2024-11-13T05:41:57,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:57,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:57,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-13T05:41:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-13T05:41:57,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-13T05:41:57,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8940 sec 2024-11-13T05:41:57,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.8970 sec 2024-11-13T05:41:58,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:41:58,055 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:41:58,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:41:58,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/56b8f715d9be4f888f18baca168a3907 is 50, key is test_row_0/A:col10/1731476518041/Put/seqid=0 2024-11-13T05:41:58,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742323_1499 (size=16931) 2024-11-13T05:41:58,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476578121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:58,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:58,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476578121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:58,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:58,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476578123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:58,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:58,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476578226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:58,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476578236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476578236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:58,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476578356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:58,363 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in 666f8f0c32525e1ac64394208a6558b0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:58,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476578446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:58,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476578456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476578456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:58,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/56b8f715d9be4f888f18baca168a3907 2024-11-13T05:41:58,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/3a256a081a2a4102be0dbf63e7a9f126 is 50, key is test_row_0/B:col10/1731476518041/Put/seqid=0 2024-11-13T05:41:58,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742324_1500 (size=12151) 2024-11-13T05:41:58,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476578756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:58,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476578767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:58,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476578767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:58,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/3a256a081a2a4102be0dbf63e7a9f126 2024-11-13T05:41:58,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/b9bfeb0dd0354075aa850c07aa0de9d2 is 50, key is test_row_0/C:col10/1731476518041/Put/seqid=0 2024-11-13T05:41:58,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742325_1501 (size=12151) 2024-11-13T05:41:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T05:41:59,117 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:41:59,117 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:41:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-13T05:41:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T05:41:59,119 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:41:59,119 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:41:59,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:41:59,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:59,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476579137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:59,144 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in 666f8f0c32525e1ac64394208a6558b0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:41:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T05:41:59,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T05:41:59,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:59,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:41:59,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:59,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:59,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:41:59,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476579276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:59,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:59,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476579276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:59,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:59,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:59,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:59,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:59,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:59,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/b9bfeb0dd0354075aa850c07aa0de9d2 2024-11-13T05:41:59,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:41:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476579286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:41:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:41:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:41:59,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/56b8f715d9be4f888f18baca168a3907 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/56b8f715d9be4f888f18baca168a3907 2024-11-13T05:41:59,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/56b8f715d9be4f888f18baca168a3907, entries=250, sequenceid=232, filesize=16.5 K 2024-11-13T05:41:59,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/3a256a081a2a4102be0dbf63e7a9f126 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/3a256a081a2a4102be0dbf63e7a9f126 2024-11-13T05:41:59,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/3a256a081a2a4102be0dbf63e7a9f126, entries=150, sequenceid=232, filesize=11.9 K 2024-11-13T05:41:59,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/b9bfeb0dd0354075aa850c07aa0de9d2 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/b9bfeb0dd0354075aa850c07aa0de9d2 2024-11-13T05:41:59,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/b9bfeb0dd0354075aa850c07aa0de9d2, entries=150, sequenceid=232, filesize=11.9 K 2024-11-13T05:41:59,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 666f8f0c32525e1ac64394208a6558b0 in 1249ms, sequenceid=232, compaction requested=true 2024-11-13T05:41:59,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:41:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:41:59,305 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:41:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:41:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:59,305 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:41:59,305 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56218 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:41:59,305 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:41:59,305 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/A is initiating minor compaction (all files) 2024-11-13T05:41:59,305 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/B is initiating minor compaction (all files) 2024-11-13T05:41:59,305 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/B in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:59,305 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/A in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:59,306 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/3a8a965b14bc46839b97085a2152a5dd, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/fd774faf06a04891a611d5cb8bbe058a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/98e3bce2cf364241838ddd5aa68e1cf8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/3a256a081a2a4102be0dbf63e7a9f126] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=47.9 K 2024-11-13T05:41:59,306 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/c984d602c73144fab16dcfe967e8e9ca, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f457b200ecab48458e1afdbb6003229b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/5becf385c2764aee94bd9288e614dad9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/56b8f715d9be4f888f18baca168a3907] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=54.9 K 2024-11-13T05:41:59,306 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting c984d602c73144fab16dcfe967e8e9ca, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731476513832 2024-11-13T05:41:59,306 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a8a965b14bc46839b97085a2152a5dd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731476513832 2024-11-13T05:41:59,306 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting f457b200ecab48458e1afdbb6003229b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731476514926 2024-11-13T05:41:59,306 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting fd774faf06a04891a611d5cb8bbe058a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731476514926 2024-11-13T05:41:59,306 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5becf385c2764aee94bd9288e614dad9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731476516135 2024-11-13T05:41:59,306 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 98e3bce2cf364241838ddd5aa68e1cf8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731476516135 2024-11-13T05:41:59,306 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56b8f715d9be4f888f18baca168a3907, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731476518041 2024-11-13T05:41:59,306 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a256a081a2a4102be0dbf63e7a9f126, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731476518041 2024-11-13T05:41:59,312 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:59,312 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#A#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:59,312 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/ff7153bc36a94e1783daaebc4a76fe03 is 50, key is test_row_0/B:col10/1731476518041/Put/seqid=0 2024-11-13T05:41:59,312 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/a4ffd8c2df174430be505f53fc6f5dc5 is 50, key is test_row_0/A:col10/1731476518041/Put/seqid=0 2024-11-13T05:41:59,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742326_1502 (size=12731) 2024-11-13T05:41:59,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742327_1503 (size=12731) 2024-11-13T05:41:59,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-13T05:41:59,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:59,424 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T05:41:59,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/09bc98efb8fb4f3e9208ecf41f7c6a09 is 50, key is test_row_0/A:col10/1731476518122/Put/seqid=0 2024-11-13T05:41:59,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742328_1504 (size=12151) 2024-11-13T05:41:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T05:41:59,722 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/a4ffd8c2df174430be505f53fc6f5dc5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/a4ffd8c2df174430be505f53fc6f5dc5 2024-11-13T05:41:59,724 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/ff7153bc36a94e1783daaebc4a76fe03 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/ff7153bc36a94e1783daaebc4a76fe03 2024-11-13T05:41:59,734 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/A of 666f8f0c32525e1ac64394208a6558b0 into a4ffd8c2df174430be505f53fc6f5dc5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:59,735 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:59,735 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/A, priority=12, startTime=1731476519304; duration=0sec 2024-11-13T05:41:59,735 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:41:59,735 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:A 2024-11-13T05:41:59,735 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:41:59,736 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/B of 666f8f0c32525e1ac64394208a6558b0 into ff7153bc36a94e1783daaebc4a76fe03(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:41:59,736 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:41:59,736 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/B, priority=12, startTime=1731476519305; duration=0sec 2024-11-13T05:41:59,736 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:41:59,736 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:B 2024-11-13T05:41:59,737 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:41:59,737 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/C is initiating minor compaction (all files) 2024-11-13T05:41:59,737 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/C in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:41:59,737 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/45aa94fa945a4a6eaef6e1579f455180, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e3e8a4c8ea214501b01efd097ef8c21e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/125d3cb3b11245b3b57530a6c2de73b3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/b9bfeb0dd0354075aa850c07aa0de9d2] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=47.9 K 2024-11-13T05:41:59,737 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45aa94fa945a4a6eaef6e1579f455180, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731476513832 2024-11-13T05:41:59,738 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3e8a4c8ea214501b01efd097ef8c21e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731476514926 2024-11-13T05:41:59,738 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 125d3cb3b11245b3b57530a6c2de73b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731476516135 2024-11-13T05:41:59,738 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9bfeb0dd0354075aa850c07aa0de9d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731476518041 2024-11-13T05:41:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T05:41:59,749 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#C#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:41:59,750 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/635ef4f332a3480a81e4e3fe8c91318f is 50, key is test_row_0/C:col10/1731476518041/Put/seqid=0 2024-11-13T05:41:59,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742329_1505 (size=12731) 2024-11-13T05:41:59,844 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/09bc98efb8fb4f3e9208ecf41f7c6a09 2024-11-13T05:41:59,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/5d0f963bf47f424392b530465a73bc12 is 50, key is test_row_0/B:col10/1731476518122/Put/seqid=0 2024-11-13T05:41:59,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742330_1506 (size=12151) 2024-11-13T05:42:00,164 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/635ef4f332a3480a81e4e3fe8c91318f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/635ef4f332a3480a81e4e3fe8c91318f 2024-11-13T05:42:00,167 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/C of 666f8f0c32525e1ac64394208a6558b0 into 635ef4f332a3480a81e4e3fe8c91318f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:00,168 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:42:00,168 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/C, priority=12, startTime=1731476519305; duration=0sec 2024-11-13T05:42:00,168 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:00,168 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:C 2024-11-13T05:42:00,256 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/5d0f963bf47f424392b530465a73bc12 2024-11-13T05:42:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T05:42:00,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/460a0364796247a79a73f471279f329e is 50, key is test_row_0/C:col10/1731476518122/Put/seqid=0 2024-11-13T05:42:00,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742331_1507 (size=12151) 2024-11-13T05:42:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:42:00,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:42:00,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476580313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:00,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476580317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:00,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476580318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:00,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34708 deadline: 1731476580386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:00,391 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_1' in 666f8f0c32525e1ac64394208a6558b0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:42:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:00,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476580426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:00,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476580426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476580436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:00,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476580636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:00,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476580636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:00,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476580647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:00,665 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/460a0364796247a79a73f471279f329e 2024-11-13T05:42:00,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/09bc98efb8fb4f3e9208ecf41f7c6a09 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/09bc98efb8fb4f3e9208ecf41f7c6a09 2024-11-13T05:42:00,671 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/09bc98efb8fb4f3e9208ecf41f7c6a09, entries=150, sequenceid=256, filesize=11.9 K 2024-11-13T05:42:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/5d0f963bf47f424392b530465a73bc12 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/5d0f963bf47f424392b530465a73bc12 2024-11-13T05:42:00,674 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/5d0f963bf47f424392b530465a73bc12, entries=150, sequenceid=256, filesize=11.9 K 2024-11-13T05:42:00,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/460a0364796247a79a73f471279f329e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/460a0364796247a79a73f471279f329e 2024-11-13T05:42:00,677 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/460a0364796247a79a73f471279f329e, entries=150, sequenceid=256, filesize=11.9 K 2024-11-13T05:42:00,678 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 666f8f0c32525e1ac64394208a6558b0 in 1254ms, sequenceid=256, compaction requested=false 2024-11-13T05:42:00,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:42:00,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:00,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-13T05:42:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-13T05:42:00,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-13T05:42:00,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5600 sec 2024-11-13T05:42:00,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.5640 sec 2024-11-13T05:42:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:42:00,951 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:42:00,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T05:42:00,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/1c2e587ca63e4147a587ad44e07cea39 is 50, key is test_row_0/A:col10/1731476520315/Put/seqid=0 2024-11-13T05:42:00,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742332_1508 (size=14741) 2024-11-13T05:42:01,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476581007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:01,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476581008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:01,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476581012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:01,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476581117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:01,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:01,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476581126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:01,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476581126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T05:42:01,266 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:01,267 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:42:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-13T05:42:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T05:42:01,268 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:42:01,269 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:42:01,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:42:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476581326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:01,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476581337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476581337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:01,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/1c2e587ca63e4147a587ad44e07cea39 2024-11-13T05:42:01,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/1bff760d7cbd4150876ae92e1f8d6880 is 50, key is test_row_0/B:col10/1731476520315/Put/seqid=0 2024-11-13T05:42:01,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742333_1509 (size=12301) 2024-11-13T05:42:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T05:42:01,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T05:42:01,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:01,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:42:01,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:01,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:01,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:01,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T05:42:01,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:01,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:42:01,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:01,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:01,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T05:42:01,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476581646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:01,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476581656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:01,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476581657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:01,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T05:42:01,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:01,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:42:01,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:01,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:01,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:01,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/1bff760d7cbd4150876ae92e1f8d6880 2024-11-13T05:42:01,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/3dc5eb82cd784eb590d9c679f4022c34 is 50, key is test_row_0/C:col10/1731476520315/Put/seqid=0 2024-11-13T05:42:01,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742334_1510 (size=12301) 2024-11-13T05:42:01,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T05:42:01,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:01,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:42:01,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:01,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:01,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T05:42:02,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T05:42:02,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:02,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:42:02,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:02,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:02,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:02,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476582166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:02,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476582166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:02,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/3dc5eb82cd784eb590d9c679f4022c34 2024-11-13T05:42:02,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/1c2e587ca63e4147a587ad44e07cea39 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/1c2e587ca63e4147a587ad44e07cea39 2024-11-13T05:42:02,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/1c2e587ca63e4147a587ad44e07cea39, entries=200, sequenceid=272, filesize=14.4 K 2024-11-13T05:42:02,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/1bff760d7cbd4150876ae92e1f8d6880 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/1bff760d7cbd4150876ae92e1f8d6880 2024-11-13T05:42:02,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476582177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:02,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T05:42:02,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:02,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:42:02,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:02,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:02,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:02,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/1bff760d7cbd4150876ae92e1f8d6880, entries=150, sequenceid=272, filesize=12.0 K 2024-11-13T05:42:02,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/3dc5eb82cd784eb590d9c679f4022c34 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/3dc5eb82cd784eb590d9c679f4022c34 2024-11-13T05:42:02,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/3dc5eb82cd784eb590d9c679f4022c34, entries=150, sequenceid=272, filesize=12.0 K 2024-11-13T05:42:02,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 666f8f0c32525e1ac64394208a6558b0 in 1239ms, sequenceid=272, compaction requested=true 2024-11-13T05:42:02,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:42:02,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:42:02,190 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:02,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:02,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:42:02,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:02,191 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:02,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:42:02,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:02,191 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:02,191 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:02,191 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/B is initiating minor compaction (all files) 2024-11-13T05:42:02,191 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/A is initiating minor compaction (all files) 2024-11-13T05:42:02,191 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/B in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:02,191 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/A in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:02,191 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/a4ffd8c2df174430be505f53fc6f5dc5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/09bc98efb8fb4f3e9208ecf41f7c6a09, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/1c2e587ca63e4147a587ad44e07cea39] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=38.7 K 2024-11-13T05:42:02,191 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/ff7153bc36a94e1783daaebc4a76fe03, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/5d0f963bf47f424392b530465a73bc12, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/1bff760d7cbd4150876ae92e1f8d6880] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=36.3 K 2024-11-13T05:42:02,192 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting ff7153bc36a94e1783daaebc4a76fe03, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731476518041 2024-11-13T05:42:02,192 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4ffd8c2df174430be505f53fc6f5dc5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731476518041 2024-11-13T05:42:02,192 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d0f963bf47f424392b530465a73bc12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731476518111 2024-11-13T05:42:02,192 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09bc98efb8fb4f3e9208ecf41f7c6a09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731476518111 2024-11-13T05:42:02,193 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c2e587ca63e4147a587ad44e07cea39, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731476520311 2024-11-13T05:42:02,193 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bff760d7cbd4150876ae92e1f8d6880, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731476520315 2024-11-13T05:42:02,198 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#A#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:02,198 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#B#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:02,198 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/1ba5cc5a8bdd4908bfd243619f8abeaf is 50, key is test_row_0/A:col10/1731476520315/Put/seqid=0 2024-11-13T05:42:02,198 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/c4b3a596456a4d6aba63e30a6970ad98 is 50, key is test_row_0/B:col10/1731476520315/Put/seqid=0 2024-11-13T05:42:02,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742335_1511 (size=12983) 2024-11-13T05:42:02,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742336_1512 (size=12983) 2024-11-13T05:42:02,209 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/1ba5cc5a8bdd4908bfd243619f8abeaf as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/1ba5cc5a8bdd4908bfd243619f8abeaf 2024-11-13T05:42:02,212 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/c4b3a596456a4d6aba63e30a6970ad98 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/c4b3a596456a4d6aba63e30a6970ad98 2024-11-13T05:42:02,214 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/A of 666f8f0c32525e1ac64394208a6558b0 into 1ba5cc5a8bdd4908bfd243619f8abeaf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:02,214 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:42:02,214 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/A, priority=13, startTime=1731476522190; duration=0sec 2024-11-13T05:42:02,214 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:02,214 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:A 2024-11-13T05:42:02,215 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:02,215 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:02,215 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/C is initiating minor compaction (all files) 2024-11-13T05:42:02,215 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/C in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:02,215 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/635ef4f332a3480a81e4e3fe8c91318f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/460a0364796247a79a73f471279f329e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/3dc5eb82cd784eb590d9c679f4022c34] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=36.3 K 2024-11-13T05:42:02,215 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 635ef4f332a3480a81e4e3fe8c91318f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731476518041 2024-11-13T05:42:02,215 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/B of 666f8f0c32525e1ac64394208a6558b0 into c4b3a596456a4d6aba63e30a6970ad98(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:02,216 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:42:02,216 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/B, priority=13, startTime=1731476522190; duration=0sec 2024-11-13T05:42:02,216 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:02,216 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 460a0364796247a79a73f471279f329e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731476518111 2024-11-13T05:42:02,216 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:B 2024-11-13T05:42:02,216 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dc5eb82cd784eb590d9c679f4022c34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731476520315 2024-11-13T05:42:02,222 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#C#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:02,222 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/541cd4404c4a4859a38757cbbeee2b90 is 50, key is test_row_0/C:col10/1731476520315/Put/seqid=0 2024-11-13T05:42:02,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742337_1513 (size=12983) 2024-11-13T05:42:02,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T05:42:02,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:02,335 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T05:42:02,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/9a939ddae354435387e9953a9718a0c8 is 50, key is test_row_0/A:col10/1731476521001/Put/seqid=0 2024-11-13T05:42:02,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742338_1514 (size=12301) 2024-11-13T05:42:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T05:42:02,629 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/541cd4404c4a4859a38757cbbeee2b90 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/541cd4404c4a4859a38757cbbeee2b90 2024-11-13T05:42:02,632 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/C of 666f8f0c32525e1ac64394208a6558b0 into 541cd4404c4a4859a38757cbbeee2b90(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:02,632 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:42:02,633 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/C, priority=13, startTime=1731476522191; duration=0sec 2024-11-13T05:42:02,633 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:02,633 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:C 2024-11-13T05:42:02,742 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/9a939ddae354435387e9953a9718a0c8 2024-11-13T05:42:02,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/2a1ed0b233bf468cbf4d1c7ab521a68b is 50, key is test_row_0/B:col10/1731476521001/Put/seqid=0 2024-11-13T05:42:02,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742339_1515 (size=12301) 2024-11-13T05:42:02,754 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/2a1ed0b233bf468cbf4d1c7ab521a68b 2024-11-13T05:42:02,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/d1837fe09e6140089d4fc7407f3898c2 is 50, key is test_row_0/C:col10/1731476521001/Put/seqid=0 2024-11-13T05:42:02,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742340_1516 (size=12301) 2024-11-13T05:42:02,774 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/d1837fe09e6140089d4fc7407f3898c2 2024-11-13T05:42:02,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/9a939ddae354435387e9953a9718a0c8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/9a939ddae354435387e9953a9718a0c8 2024-11-13T05:42:02,782 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/9a939ddae354435387e9953a9718a0c8, entries=150, sequenceid=297, filesize=12.0 K 2024-11-13T05:42:02,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/2a1ed0b233bf468cbf4d1c7ab521a68b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/2a1ed0b233bf468cbf4d1c7ab521a68b 2024-11-13T05:42:02,786 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/2a1ed0b233bf468cbf4d1c7ab521a68b, entries=150, sequenceid=297, filesize=12.0 K 2024-11-13T05:42:02,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/d1837fe09e6140089d4fc7407f3898c2 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/d1837fe09e6140089d4fc7407f3898c2 2024-11-13T05:42:02,790 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/d1837fe09e6140089d4fc7407f3898c2, entries=150, sequenceid=297, filesize=12.0 K 2024-11-13T05:42:02,791 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 666f8f0c32525e1ac64394208a6558b0 in 456ms, sequenceid=297, compaction requested=false 2024-11-13T05:42:02,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:42:02,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:02,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-13T05:42:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-13T05:42:02,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-13T05:42:02,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5230 sec 2024-11-13T05:42:02,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.5290 sec 2024-11-13T05:42:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:42:03,197 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:42:03,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:42:03,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/6cdc56db3e284ea5962c80cdf97af4e4 is 50, key is test_row_0/A:col10/1731476523192/Put/seqid=0 2024-11-13T05:42:03,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742341_1517 (size=19621) 2024-11-13T05:42:03,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476583279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476583281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476583282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476583288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476583396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476583396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476583397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476583406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T05:42:03,417 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:03,417 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:42:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-13T05:42:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T05:42:03,419 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:42:03,419 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:42:03,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:42:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T05:42:03,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T05:42:03,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:03,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:42:03,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:03,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:03,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:03,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/6cdc56db3e284ea5962c80cdf97af4e4 2024-11-13T05:42:03,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/4de0178efb8e4589b5f337e5417ec763 is 50, key is test_row_0/B:col10/1731476523192/Put/seqid=0 2024-11-13T05:42:03,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476583606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476583606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476583606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742342_1518 (size=12301) 2024-11-13T05:42:03,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476583616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T05:42:03,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:03,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:42:03,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:03,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:03,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T05:42:03,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T05:42:03,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:03,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:42:03,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:03,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:03,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:03,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34718 deadline: 1731476583916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34724 deadline: 1731476583916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34668 deadline: 1731476583926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1731476583927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 , the old value is region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:42:04,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/4de0178efb8e4589b5f337e5417ec763 2024-11-13T05:42:04,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/9378f792ae9044b3bfac77ea70c352f3 is 50, key is test_row_0/C:col10/1731476523192/Put/seqid=0 2024-11-13T05:42:04,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742343_1519 (size=12301) 2024-11-13T05:42:04,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/9378f792ae9044b3bfac77ea70c352f3 2024-11-13T05:42:04,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/6cdc56db3e284ea5962c80cdf97af4e4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/6cdc56db3e284ea5962c80cdf97af4e4 2024-11-13T05:42:04,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T05:42:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:04,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/6cdc56db3e284ea5962c80cdf97af4e4, entries=300, sequenceid=309, filesize=19.2 K 2024-11-13T05:42:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:42:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:04,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:04,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:04,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/4de0178efb8e4589b5f337e5417ec763 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/4de0178efb8e4589b5f337e5417ec763 2024-11-13T05:42:04,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/4de0178efb8e4589b5f337e5417ec763, entries=150, sequenceid=309, filesize=12.0 K 2024-11-13T05:42:04,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/9378f792ae9044b3bfac77ea70c352f3 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/9378f792ae9044b3bfac77ea70c352f3 2024-11-13T05:42:04,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/9378f792ae9044b3bfac77ea70c352f3, entries=150, sequenceid=309, filesize=12.0 K 2024-11-13T05:42:04,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 666f8f0c32525e1ac64394208a6558b0 in 841ms, sequenceid=309, compaction requested=true 2024-11-13T05:42:04,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:42:04,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:42:04,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:04,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:42:04,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:04,038 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:04,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 666f8f0c32525e1ac64394208a6558b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:42:04,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:04,038 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:04,039 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44905 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:04,039 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:04,039 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/A is initiating minor compaction (all files) 2024-11-13T05:42:04,039 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/B is initiating minor compaction (all files) 2024-11-13T05:42:04,039 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/A in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:04,039 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/B in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:04,039 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/1ba5cc5a8bdd4908bfd243619f8abeaf, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/9a939ddae354435387e9953a9718a0c8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/6cdc56db3e284ea5962c80cdf97af4e4] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=43.9 K 2024-11-13T05:42:04,039 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/c4b3a596456a4d6aba63e30a6970ad98, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/2a1ed0b233bf468cbf4d1c7ab521a68b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/4de0178efb8e4589b5f337e5417ec763] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=36.7 K 2024-11-13T05:42:04,040 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ba5cc5a8bdd4908bfd243619f8abeaf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731476520315 2024-11-13T05:42:04,040 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting c4b3a596456a4d6aba63e30a6970ad98, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731476520315 2024-11-13T05:42:04,040 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a939ddae354435387e9953a9718a0c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731476520987 2024-11-13T05:42:04,040 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a1ed0b233bf468cbf4d1c7ab521a68b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731476520987 2024-11-13T05:42:04,040 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cdc56db3e284ea5962c80cdf97af4e4, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731476523188 2024-11-13T05:42:04,040 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 4de0178efb8e4589b5f337e5417ec763, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731476523192 2024-11-13T05:42:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T05:42:04,047 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#A#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:04,047 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/23dffe665875461f9fbb5161b69f9b68 is 50, key is test_row_0/A:col10/1731476523192/Put/seqid=0 2024-11-13T05:42:04,050 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#B#compaction#441 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:04,050 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/dd26ca3de9e647b9a5afd221aeaa55bb is 50, key is test_row_0/B:col10/1731476523192/Put/seqid=0 2024-11-13T05:42:04,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742344_1520 (size=13085) 2024-11-13T05:42:04,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742345_1521 (size=13085) 2024-11-13T05:42:04,065 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/23dffe665875461f9fbb5161b69f9b68 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/23dffe665875461f9fbb5161b69f9b68 2024-11-13T05:42:04,071 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/A of 666f8f0c32525e1ac64394208a6558b0 into 23dffe665875461f9fbb5161b69f9b68(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:04,071 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:42:04,072 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/A, priority=13, startTime=1731476524038; duration=0sec 2024-11-13T05:42:04,072 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:04,072 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:A 2024-11-13T05:42:04,072 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:04,072 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:04,072 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): 666f8f0c32525e1ac64394208a6558b0/C is initiating minor compaction (all files) 2024-11-13T05:42:04,073 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 666f8f0c32525e1ac64394208a6558b0/C in TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:04,073 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/541cd4404c4a4859a38757cbbeee2b90, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/d1837fe09e6140089d4fc7407f3898c2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/9378f792ae9044b3bfac77ea70c352f3] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp, totalSize=36.7 K 2024-11-13T05:42:04,073 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 541cd4404c4a4859a38757cbbeee2b90, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731476520315 2024-11-13T05:42:04,073 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1837fe09e6140089d4fc7407f3898c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731476520987 2024-11-13T05:42:04,073 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9378f792ae9044b3bfac77ea70c352f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731476523192 2024-11-13T05:42:04,078 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 666f8f0c32525e1ac64394208a6558b0#C#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:04,079 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/ba23b1e3786344208efefbafadc853f4 is 50, key is test_row_0/C:col10/1731476523192/Put/seqid=0 2024-11-13T05:42:04,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742346_1522 (size=13085) 2024-11-13T05:42:04,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T05:42:04,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:04,183 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T05:42:04,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/6d11b29c2bf649f8bb22a8bf01f67664 is 50, key is test_row_0/A:col10/1731476523287/Put/seqid=0 2024-11-13T05:42:04,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742347_1523 (size=12301) 2024-11-13T05:42:04,359 INFO [Thread-2031 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2031. 2024-11-13T05:42:04,359 DEBUG [Thread-2031 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:04,359 DEBUG [Thread-2031 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,360 DEBUG [Thread-2031 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,360 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:04,360 INFO [Thread-2033 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2033. 2024-11-13T05:42:04,360 DEBUG [Thread-2033 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:04,360 DEBUG [Thread-2033 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,360 DEBUG [Thread-2033 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,360 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:04,362 INFO [Thread-2035 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2035. 2024-11-13T05:42:04,362 DEBUG [Thread-2035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:04,362 DEBUG [Thread-2035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,362 DEBUG [Thread-2035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,362 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:04,365 INFO [Thread-2027 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2027. 2024-11-13T05:42:04,365 DEBUG [Thread-2027 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:04,365 DEBUG [Thread-2027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,365 DEBUG [Thread-2027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:04,366 INFO [Thread-2029 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2029. 2024-11-13T05:42:04,366 DEBUG [Thread-2029 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:04,366 DEBUG [Thread-2029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,366 DEBUG [Thread-2029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:42:04,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. as already flushing 2024-11-13T05:42:04,430 INFO [Thread-2018 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2018. 2024-11-13T05:42:04,430 INFO [Thread-2024 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2024. 2024-11-13T05:42:04,430 DEBUG [Thread-2018 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:04,430 DEBUG [Thread-2018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,430 DEBUG [Thread-2024 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:04,430 DEBUG [Thread-2024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,431 DEBUG [Thread-2018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,431 DEBUG [Thread-2024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,431 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:04,431 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:04,439 INFO [Thread-2016 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2016. 2024-11-13T05:42:04,439 INFO [Thread-2022 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2022. 2024-11-13T05:42:04,439 DEBUG [Thread-2016 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:04,439 DEBUG [Thread-2016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,439 DEBUG [Thread-2022 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:04,439 DEBUG [Thread-2022 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,439 DEBUG [Thread-2016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:04,439 DEBUG [Thread-2022 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:04,447 INFO [Thread-2020 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2020. 2024-11-13T05:42:04,447 DEBUG [Thread-2020 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:04,447 DEBUG [Thread-2020 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,448 DEBUG [Thread-2020 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:04,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:04,470 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/dd26ca3de9e647b9a5afd221aeaa55bb as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/dd26ca3de9e647b9a5afd221aeaa55bb 2024-11-13T05:42:04,476 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/B of 666f8f0c32525e1ac64394208a6558b0 into dd26ca3de9e647b9a5afd221aeaa55bb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:04,476 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:42:04,476 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/B, priority=13, startTime=1731476524038; duration=0sec 2024-11-13T05:42:04,477 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:04,477 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:B 2024-11-13T05:42:04,487 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/ba23b1e3786344208efefbafadc853f4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/ba23b1e3786344208efefbafadc853f4 2024-11-13T05:42:04,494 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 666f8f0c32525e1ac64394208a6558b0/C of 666f8f0c32525e1ac64394208a6558b0 into ba23b1e3786344208efefbafadc853f4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:04,494 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:42:04,494 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., storeName=666f8f0c32525e1ac64394208a6558b0/C, priority=13, startTime=1731476524038; duration=0sec 2024-11-13T05:42:04,494 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:04,494 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 666f8f0c32525e1ac64394208a6558b0:C 2024-11-13T05:42:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T05:42:04,590 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/6d11b29c2bf649f8bb22a8bf01f67664 2024-11-13T05:42:04,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/1e1c2ded836b4f289626e3f1ca42fab1 is 50, key is test_row_0/B:col10/1731476523287/Put/seqid=0 2024-11-13T05:42:04,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742348_1524 (size=12301) 2024-11-13T05:42:05,006 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/1e1c2ded836b4f289626e3f1ca42fab1 2024-11-13T05:42:05,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/c73e4f6ccf2a4b859e7f81579182b358 is 50, key is test_row_0/C:col10/1731476523287/Put/seqid=0 2024-11-13T05:42:05,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742349_1525 (size=12301) 2024-11-13T05:42:05,452 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/c73e4f6ccf2a4b859e7f81579182b358 2024-11-13T05:42:05,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/6d11b29c2bf649f8bb22a8bf01f67664 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/6d11b29c2bf649f8bb22a8bf01f67664 2024-11-13T05:42:05,467 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/6d11b29c2bf649f8bb22a8bf01f67664, entries=150, sequenceid=335, filesize=12.0 K 2024-11-13T05:42:05,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/1e1c2ded836b4f289626e3f1ca42fab1 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/1e1c2ded836b4f289626e3f1ca42fab1 2024-11-13T05:42:05,470 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/1e1c2ded836b4f289626e3f1ca42fab1, entries=150, sequenceid=335, filesize=12.0 K 2024-11-13T05:42:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/c73e4f6ccf2a4b859e7f81579182b358 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/c73e4f6ccf2a4b859e7f81579182b358 2024-11-13T05:42:05,474 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/c73e4f6ccf2a4b859e7f81579182b358, entries=150, sequenceid=335, filesize=12.0 K 2024-11-13T05:42:05,474 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=33.54 KB/34350 for 666f8f0c32525e1ac64394208a6558b0 in 1291ms, sequenceid=335, compaction requested=false 2024-11-13T05:42:05,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 666f8f0c32525e1ac64394208a6558b0: 2024-11-13T05:42:05,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:05,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-13T05:42:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-13T05:42:05,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-13T05:42:05,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0570 sec 2024-11-13T05:42:05,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.0600 sec 2024-11-13T05:42:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T05:42:05,567 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:05,567 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1667 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4993 rows 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1673 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5015 rows 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1666 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4997 rows 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1660 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4980 rows 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1663 2024-11-13T05:42:05,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4982 rows 2024-11-13T05:42:05,568 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:42:05,568 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:42:05,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:05,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:05,569 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:05,570 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T05:42:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T05:42:05,575 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476525575"}]},"ts":"1731476525575"} 2024-11-13T05:42:05,577 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T05:42:05,577 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T05:42:05,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:42:05,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=666f8f0c32525e1ac64394208a6558b0, UNASSIGN}] 2024-11-13T05:42:05,582 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=666f8f0c32525e1ac64394208a6558b0, UNASSIGN 2024-11-13T05:42:05,583 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=666f8f0c32525e1ac64394208a6558b0, regionState=CLOSING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:05,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=666f8f0c32525e1ac64394208a6558b0, UNASSIGN because future has completed 2024-11-13T05:42:05,585 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:42:05,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:42:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T05:42:05,738 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(122): Close 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:42:05,738 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:42:05,738 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1722): Closing 666f8f0c32525e1ac64394208a6558b0, disabling compactions & flushes 2024-11-13T05:42:05,738 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:05,739 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:05,739 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. after waiting 0 ms 2024-11-13T05:42:05,739 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:05,739 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(2902): Flushing 666f8f0c32525e1ac64394208a6558b0 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-13T05:42:05,744 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/e3639aaa335c40928bce2e8ec03761c6 is 50, key is test_row_0/A:col10/1731476524437/Put/seqid=0 2024-11-13T05:42:05,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742350_1526 (size=9857) 2024-11-13T05:42:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T05:42:06,149 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/e3639aaa335c40928bce2e8ec03761c6 2024-11-13T05:42:06,162 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/1eb40175f6774aaa8286f5ab13610e9b is 50, key is test_row_0/B:col10/1731476524437/Put/seqid=0 2024-11-13T05:42:06,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742351_1527 (size=9857) 2024-11-13T05:42:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T05:42:06,568 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/1eb40175f6774aaa8286f5ab13610e9b 2024-11-13T05:42:06,580 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/fff649383977414dae4418534d280541 is 50, key is test_row_0/C:col10/1731476524437/Put/seqid=0 2024-11-13T05:42:06,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742352_1528 (size=9857) 2024-11-13T05:42:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T05:42:06,985 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/fff649383977414dae4418534d280541 2024-11-13T05:42:06,996 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/A/e3639aaa335c40928bce2e8ec03761c6 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/e3639aaa335c40928bce2e8ec03761c6 2024-11-13T05:42:07,002 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/e3639aaa335c40928bce2e8ec03761c6, entries=100, sequenceid=345, filesize=9.6 K 2024-11-13T05:42:07,004 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/B/1eb40175f6774aaa8286f5ab13610e9b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/1eb40175f6774aaa8286f5ab13610e9b 2024-11-13T05:42:07,009 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/1eb40175f6774aaa8286f5ab13610e9b, entries=100, sequenceid=345, filesize=9.6 K 2024-11-13T05:42:07,010 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/.tmp/C/fff649383977414dae4418534d280541 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/fff649383977414dae4418534d280541 2024-11-13T05:42:07,014 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/fff649383977414dae4418534d280541, entries=100, sequenceid=345, filesize=9.6 K 2024-11-13T05:42:07,015 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 666f8f0c32525e1ac64394208a6558b0 in 1276ms, sequenceid=345, compaction requested=true 2024-11-13T05:42:07,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/db2be2f3a5e44dc98a8b1dcc9c3facbd, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/ed48e6a03ab54060a689beb98f11a4d6, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/acd5d450985e4c60bef8f89260f63944, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f07ccc20d8f24e88af6c1a6ca73e60c5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/adecc36bbf4346928bbdc8e48e8b60ff, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/a6eb37b3e79749ed8e6626ab70a23c1a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/acb7255f76e64bf69ae280b22c51b1c3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/12f51dcc1f9d44c09774b30ca5096fd3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/05fdbe9c94bb4e9bbc52a35a33245925, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/87937115aead4bc9a2014cd89d19dabd, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f0acad97281e4b01a7f8267d62b94bbb, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/7260c098e707400fb465992903fbbd98, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/c82c0bfe2620430ca3ff8ad46794ae70, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/c984d602c73144fab16dcfe967e8e9ca, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f457b200ecab48458e1afdbb6003229b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/5becf385c2764aee94bd9288e614dad9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/56b8f715d9be4f888f18baca168a3907, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/a4ffd8c2df174430be505f53fc6f5dc5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/09bc98efb8fb4f3e9208ecf41f7c6a09, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/1c2e587ca63e4147a587ad44e07cea39, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/1ba5cc5a8bdd4908bfd243619f8abeaf, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/9a939ddae354435387e9953a9718a0c8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/6cdc56db3e284ea5962c80cdf97af4e4] to archive 2024-11-13T05:42:07,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:42:07,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/db2be2f3a5e44dc98a8b1dcc9c3facbd to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/db2be2f3a5e44dc98a8b1dcc9c3facbd 2024-11-13T05:42:07,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/ed48e6a03ab54060a689beb98f11a4d6 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/ed48e6a03ab54060a689beb98f11a4d6 2024-11-13T05:42:07,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/acd5d450985e4c60bef8f89260f63944 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/acd5d450985e4c60bef8f89260f63944 2024-11-13T05:42:07,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f07ccc20d8f24e88af6c1a6ca73e60c5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f07ccc20d8f24e88af6c1a6ca73e60c5 2024-11-13T05:42:07,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/adecc36bbf4346928bbdc8e48e8b60ff to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/adecc36bbf4346928bbdc8e48e8b60ff 2024-11-13T05:42:07,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/a6eb37b3e79749ed8e6626ab70a23c1a to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/a6eb37b3e79749ed8e6626ab70a23c1a 2024-11-13T05:42:07,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/acb7255f76e64bf69ae280b22c51b1c3 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/acb7255f76e64bf69ae280b22c51b1c3 2024-11-13T05:42:07,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/12f51dcc1f9d44c09774b30ca5096fd3 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/12f51dcc1f9d44c09774b30ca5096fd3 2024-11-13T05:42:07,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/05fdbe9c94bb4e9bbc52a35a33245925 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/05fdbe9c94bb4e9bbc52a35a33245925 2024-11-13T05:42:07,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/87937115aead4bc9a2014cd89d19dabd to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/87937115aead4bc9a2014cd89d19dabd 2024-11-13T05:42:07,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f0acad97281e4b01a7f8267d62b94bbb to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f0acad97281e4b01a7f8267d62b94bbb 2024-11-13T05:42:07,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/7260c098e707400fb465992903fbbd98 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/7260c098e707400fb465992903fbbd98 2024-11-13T05:42:07,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/c82c0bfe2620430ca3ff8ad46794ae70 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/c82c0bfe2620430ca3ff8ad46794ae70 2024-11-13T05:42:07,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/c984d602c73144fab16dcfe967e8e9ca to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/c984d602c73144fab16dcfe967e8e9ca 2024-11-13T05:42:07,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f457b200ecab48458e1afdbb6003229b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/f457b200ecab48458e1afdbb6003229b 2024-11-13T05:42:07,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/5becf385c2764aee94bd9288e614dad9 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/5becf385c2764aee94bd9288e614dad9 2024-11-13T05:42:07,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/56b8f715d9be4f888f18baca168a3907 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/56b8f715d9be4f888f18baca168a3907 2024-11-13T05:42:07,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/a4ffd8c2df174430be505f53fc6f5dc5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/a4ffd8c2df174430be505f53fc6f5dc5 2024-11-13T05:42:07,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/09bc98efb8fb4f3e9208ecf41f7c6a09 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/09bc98efb8fb4f3e9208ecf41f7c6a09 2024-11-13T05:42:07,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/1c2e587ca63e4147a587ad44e07cea39 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/1c2e587ca63e4147a587ad44e07cea39 2024-11-13T05:42:07,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/1ba5cc5a8bdd4908bfd243619f8abeaf to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/1ba5cc5a8bdd4908bfd243619f8abeaf 2024-11-13T05:42:07,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/9a939ddae354435387e9953a9718a0c8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/9a939ddae354435387e9953a9718a0c8 2024-11-13T05:42:07,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/6cdc56db3e284ea5962c80cdf97af4e4 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/6cdc56db3e284ea5962c80cdf97af4e4 2024-11-13T05:42:07,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/20c0654a214c4de3a8105283c1642e65, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/c233d1d5e7c446378146abd9740416ea, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/7f927d8b45c047ad964c7400e9d8840e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/4d958dd41a4d44ea986e171add0797f5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/cc79574c228f41a2b9060a604bc89389, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/7c9c3d991cd34a92911ee84799ee715e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/2d4c9032ff724a72825056c9a2b6a468, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/d9a867e2241a4032924c0bfe7f6e864e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/400858ea25604d9389fbfa948ad8c582, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/63bb438a9dfd40e888ff13b789004e56, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/a402f182caec4b15a94d75972f0bd7b1, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/24cc9bc17baa4cbaa5511e3a2f835ae9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/3a8a965b14bc46839b97085a2152a5dd, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/56ab6fa3ebb54afc8b78832c46cb5907, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/fd774faf06a04891a611d5cb8bbe058a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/98e3bce2cf364241838ddd5aa68e1cf8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/ff7153bc36a94e1783daaebc4a76fe03, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/3a256a081a2a4102be0dbf63e7a9f126, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/5d0f963bf47f424392b530465a73bc12, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/c4b3a596456a4d6aba63e30a6970ad98, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/1bff760d7cbd4150876ae92e1f8d6880, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/2a1ed0b233bf468cbf4d1c7ab521a68b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/4de0178efb8e4589b5f337e5417ec763] to archive 2024-11-13T05:42:07,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:42:07,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/20c0654a214c4de3a8105283c1642e65 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/20c0654a214c4de3a8105283c1642e65 2024-11-13T05:42:07,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/c233d1d5e7c446378146abd9740416ea to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/c233d1d5e7c446378146abd9740416ea 2024-11-13T05:42:07,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/7f927d8b45c047ad964c7400e9d8840e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/7f927d8b45c047ad964c7400e9d8840e 2024-11-13T05:42:07,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/4d958dd41a4d44ea986e171add0797f5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/4d958dd41a4d44ea986e171add0797f5 2024-11-13T05:42:07,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/cc79574c228f41a2b9060a604bc89389 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/cc79574c228f41a2b9060a604bc89389 2024-11-13T05:42:07,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/7c9c3d991cd34a92911ee84799ee715e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/7c9c3d991cd34a92911ee84799ee715e 2024-11-13T05:42:07,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/2d4c9032ff724a72825056c9a2b6a468 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/2d4c9032ff724a72825056c9a2b6a468 2024-11-13T05:42:07,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/d9a867e2241a4032924c0bfe7f6e864e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/d9a867e2241a4032924c0bfe7f6e864e 2024-11-13T05:42:07,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/400858ea25604d9389fbfa948ad8c582 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/400858ea25604d9389fbfa948ad8c582 2024-11-13T05:42:07,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/63bb438a9dfd40e888ff13b789004e56 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/63bb438a9dfd40e888ff13b789004e56 2024-11-13T05:42:07,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/a402f182caec4b15a94d75972f0bd7b1 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/a402f182caec4b15a94d75972f0bd7b1 2024-11-13T05:42:07,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/24cc9bc17baa4cbaa5511e3a2f835ae9 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/24cc9bc17baa4cbaa5511e3a2f835ae9 2024-11-13T05:42:07,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/3a8a965b14bc46839b97085a2152a5dd to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/3a8a965b14bc46839b97085a2152a5dd 2024-11-13T05:42:07,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/56ab6fa3ebb54afc8b78832c46cb5907 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/56ab6fa3ebb54afc8b78832c46cb5907 2024-11-13T05:42:07,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/fd774faf06a04891a611d5cb8bbe058a to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/fd774faf06a04891a611d5cb8bbe058a 2024-11-13T05:42:07,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/98e3bce2cf364241838ddd5aa68e1cf8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/98e3bce2cf364241838ddd5aa68e1cf8 2024-11-13T05:42:07,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/ff7153bc36a94e1783daaebc4a76fe03 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/ff7153bc36a94e1783daaebc4a76fe03 2024-11-13T05:42:07,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/3a256a081a2a4102be0dbf63e7a9f126 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/3a256a081a2a4102be0dbf63e7a9f126 2024-11-13T05:42:07,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/5d0f963bf47f424392b530465a73bc12 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/5d0f963bf47f424392b530465a73bc12 2024-11-13T05:42:07,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/c4b3a596456a4d6aba63e30a6970ad98 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/c4b3a596456a4d6aba63e30a6970ad98 2024-11-13T05:42:07,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/1bff760d7cbd4150876ae92e1f8d6880 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/1bff760d7cbd4150876ae92e1f8d6880 2024-11-13T05:42:07,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/2a1ed0b233bf468cbf4d1c7ab521a68b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/2a1ed0b233bf468cbf4d1c7ab521a68b 2024-11-13T05:42:07,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/4de0178efb8e4589b5f337e5417ec763 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/4de0178efb8e4589b5f337e5417ec763 2024-11-13T05:42:07,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e7f7d7d8f6e54557b64169f9861f4b47, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e9a84bed330f41a29bc5f9038d495126, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/956caf6f5e76427aa349d98e699919dc, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/a000ca8034934c19aa770703acf2ec43, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/0f645041c8344a699c34a6cd30c04296, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/bcd1e0eb8af74d69936ceb2c2f5d8b5e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/0ec1c4ee1a29454fbe2b963ce84be3ae, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/74ae11e787f14db98527bbc19932c1bd, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/08035b27da8a4440bad2d8476f734fa8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/66daab82e04542edb5cffc98015e8500, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/8e24181d40a8428fa45c74aaabf1b193, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/c2ba2d77d60b40748815b8506720dbc9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/45aa94fa945a4a6eaef6e1579f455180, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/71f2e1ebea3d4d57b40e8f1832c33e27, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e3e8a4c8ea214501b01efd097ef8c21e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/125d3cb3b11245b3b57530a6c2de73b3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/635ef4f332a3480a81e4e3fe8c91318f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/b9bfeb0dd0354075aa850c07aa0de9d2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/460a0364796247a79a73f471279f329e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/541cd4404c4a4859a38757cbbeee2b90, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/3dc5eb82cd784eb590d9c679f4022c34, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/d1837fe09e6140089d4fc7407f3898c2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/9378f792ae9044b3bfac77ea70c352f3] to archive 2024-11-13T05:42:07,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:42:07,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e7f7d7d8f6e54557b64169f9861f4b47 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e7f7d7d8f6e54557b64169f9861f4b47 2024-11-13T05:42:07,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e9a84bed330f41a29bc5f9038d495126 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e9a84bed330f41a29bc5f9038d495126 2024-11-13T05:42:07,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/956caf6f5e76427aa349d98e699919dc to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/956caf6f5e76427aa349d98e699919dc 2024-11-13T05:42:07,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/a000ca8034934c19aa770703acf2ec43 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/a000ca8034934c19aa770703acf2ec43 2024-11-13T05:42:07,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/0f645041c8344a699c34a6cd30c04296 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/0f645041c8344a699c34a6cd30c04296 2024-11-13T05:42:07,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/bcd1e0eb8af74d69936ceb2c2f5d8b5e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/bcd1e0eb8af74d69936ceb2c2f5d8b5e 2024-11-13T05:42:07,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/0ec1c4ee1a29454fbe2b963ce84be3ae to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/0ec1c4ee1a29454fbe2b963ce84be3ae 2024-11-13T05:42:07,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/74ae11e787f14db98527bbc19932c1bd to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/74ae11e787f14db98527bbc19932c1bd 2024-11-13T05:42:07,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/08035b27da8a4440bad2d8476f734fa8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/08035b27da8a4440bad2d8476f734fa8 2024-11-13T05:42:07,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/66daab82e04542edb5cffc98015e8500 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/66daab82e04542edb5cffc98015e8500 2024-11-13T05:42:07,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/8e24181d40a8428fa45c74aaabf1b193 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/8e24181d40a8428fa45c74aaabf1b193 2024-11-13T05:42:07,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/c2ba2d77d60b40748815b8506720dbc9 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/c2ba2d77d60b40748815b8506720dbc9 2024-11-13T05:42:07,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/45aa94fa945a4a6eaef6e1579f455180 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/45aa94fa945a4a6eaef6e1579f455180 2024-11-13T05:42:07,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/71f2e1ebea3d4d57b40e8f1832c33e27 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/71f2e1ebea3d4d57b40e8f1832c33e27 2024-11-13T05:42:07,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e3e8a4c8ea214501b01efd097ef8c21e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/e3e8a4c8ea214501b01efd097ef8c21e 2024-11-13T05:42:07,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/125d3cb3b11245b3b57530a6c2de73b3 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/125d3cb3b11245b3b57530a6c2de73b3 2024-11-13T05:42:07,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/635ef4f332a3480a81e4e3fe8c91318f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/635ef4f332a3480a81e4e3fe8c91318f 2024-11-13T05:42:07,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/b9bfeb0dd0354075aa850c07aa0de9d2 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/b9bfeb0dd0354075aa850c07aa0de9d2 2024-11-13T05:42:07,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/460a0364796247a79a73f471279f329e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/460a0364796247a79a73f471279f329e 2024-11-13T05:42:07,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/541cd4404c4a4859a38757cbbeee2b90 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/541cd4404c4a4859a38757cbbeee2b90 2024-11-13T05:42:07,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/3dc5eb82cd784eb590d9c679f4022c34 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/3dc5eb82cd784eb590d9c679f4022c34 2024-11-13T05:42:07,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/d1837fe09e6140089d4fc7407f3898c2 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/d1837fe09e6140089d4fc7407f3898c2 2024-11-13T05:42:07,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/9378f792ae9044b3bfac77ea70c352f3 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/9378f792ae9044b3bfac77ea70c352f3 2024-11-13T05:42:07,113 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/recovered.edits/348.seqid, newMaxSeqId=348, maxSeqId=1 2024-11-13T05:42:07,114 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. 2024-11-13T05:42:07,114 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1676): Region close journal for 666f8f0c32525e1ac64394208a6558b0: Waiting for close lock at 1731476525738Running coprocessor pre-close hooks at 1731476525738Disabling compacts and flushes for region at 1731476525738Disabling writes for close at 1731476525739 (+1 ms)Obtaining lock to block concurrent updates at 1731476525739Preparing flush snapshotting stores in 666f8f0c32525e1ac64394208a6558b0 at 1731476525739Finished memstore snapshotting TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731476525739Flushing stores of TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0. at 1731476525740 (+1 ms)Flushing 666f8f0c32525e1ac64394208a6558b0/A: creating writer at 1731476525740Flushing 666f8f0c32525e1ac64394208a6558b0/A: appending metadata at 1731476525743 (+3 ms)Flushing 666f8f0c32525e1ac64394208a6558b0/A: closing flushed file at 1731476525743Flushing 666f8f0c32525e1ac64394208a6558b0/B: creating writer at 1731476526159 (+416 ms)Flushing 666f8f0c32525e1ac64394208a6558b0/B: appending metadata at 1731476526162 (+3 ms)Flushing 666f8f0c32525e1ac64394208a6558b0/B: closing flushed file at 1731476526162Flushing 666f8f0c32525e1ac64394208a6558b0/C: creating writer at 1731476526576 (+414 ms)Flushing 666f8f0c32525e1ac64394208a6558b0/C: appending metadata at 1731476526579 (+3 ms)Flushing 666f8f0c32525e1ac64394208a6558b0/C: closing flushed file at 1731476526579Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@660cc142: reopening flushed file at 1731476526995 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46ae8547: reopening flushed file at 1731476527002 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75802f07: reopening flushed file at 1731476527009 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 666f8f0c32525e1ac64394208a6558b0 in 1276ms, sequenceid=345, compaction requested=true at 1731476527015 (+6 ms)Writing region close event to WAL at 1731476527111 (+96 ms)Running coprocessor post-close hooks at 1731476527114 (+3 ms)Closed at 1731476527114 2024-11-13T05:42:07,115 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(157): Closed 666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:42:07,116 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=666f8f0c32525e1ac64394208a6558b0, regionState=CLOSED 2024-11-13T05:42:07,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:42:07,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-13T05:42:07,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseRegionProcedure 666f8f0c32525e1ac64394208a6558b0, server=bc386b7bf2fe,45837,1731476380128 in 1.5340 sec 2024-11-13T05:42:07,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-13T05:42:07,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=666f8f0c32525e1ac64394208a6558b0, UNASSIGN in 1.5400 sec 2024-11-13T05:42:07,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-13T05:42:07,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5460 sec 2024-11-13T05:42:07,128 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476527128"}]},"ts":"1731476527128"} 2024-11-13T05:42:07,130 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T05:42:07,130 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T05:42:07,133 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5610 sec 2024-11-13T05:42:07,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T05:42:07,718 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:07,719 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T05:42:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:07,724 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T05:42:07,726 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=127, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:07,731 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:42:07,733 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/recovered.edits] 2024-11-13T05:42:07,736 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/23dffe665875461f9fbb5161b69f9b68 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/23dffe665875461f9fbb5161b69f9b68 2024-11-13T05:42:07,737 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/6d11b29c2bf649f8bb22a8bf01f67664 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/6d11b29c2bf649f8bb22a8bf01f67664 2024-11-13T05:42:07,738 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/e3639aaa335c40928bce2e8ec03761c6 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/A/e3639aaa335c40928bce2e8ec03761c6 2024-11-13T05:42:07,740 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/1e1c2ded836b4f289626e3f1ca42fab1 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/1e1c2ded836b4f289626e3f1ca42fab1 2024-11-13T05:42:07,741 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/1eb40175f6774aaa8286f5ab13610e9b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/1eb40175f6774aaa8286f5ab13610e9b 2024-11-13T05:42:07,743 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/dd26ca3de9e647b9a5afd221aeaa55bb to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/B/dd26ca3de9e647b9a5afd221aeaa55bb 2024-11-13T05:42:07,745 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/ba23b1e3786344208efefbafadc853f4 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/ba23b1e3786344208efefbafadc853f4 2024-11-13T05:42:07,747 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/c73e4f6ccf2a4b859e7f81579182b358 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/c73e4f6ccf2a4b859e7f81579182b358 2024-11-13T05:42:07,748 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/fff649383977414dae4418534d280541 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/C/fff649383977414dae4418534d280541 2024-11-13T05:42:07,752 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/recovered.edits/348.seqid to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0/recovered.edits/348.seqid 2024-11-13T05:42:07,753 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/666f8f0c32525e1ac64394208a6558b0 2024-11-13T05:42:07,753 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T05:42:07,756 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=127, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:07,758 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T05:42:07,761 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T05:42:07,762 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=127, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:07,762 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T05:42:07,763 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731476527762"}]},"ts":"9223372036854775807"} 2024-11-13T05:42:07,764 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T05:42:07,764 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 666f8f0c32525e1ac64394208a6558b0, NAME => 'TestAcidGuarantees,,1731476502139.666f8f0c32525e1ac64394208a6558b0.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T05:42:07,764 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T05:42:07,764 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731476527764"}]},"ts":"9223372036854775807"} 2024-11-13T05:42:07,766 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T05:42:07,766 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=127, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:07,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-13T05:42:07,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T05:42:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T05:42:07,837 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:07,852 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 237) - Thread LEAK? -, OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=423 (was 454), ProcessCount=11 (was 11), AvailableMemoryMB=7251 (was 7279) 2024-11-13T05:42:07,861 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=423, ProcessCount=11, AvailableMemoryMB=7251 2024-11-13T05:42:07,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:42:07,862 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:42:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:07,865 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:42:07,865 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:07,865 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 128 2024-11-13T05:42:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-13T05:42:07,866 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:42:07,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742353_1529 (size=959) 2024-11-13T05:42:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-13T05:42:08,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-13T05:42:08,278 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745 2024-11-13T05:42:08,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742354_1530 (size=53) 2024-11-13T05:42:08,392 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:42:08,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-13T05:42:08,692 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:42:08,692 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e31e188350bb5f6ecf6454cab120dda0, disabling compactions & flushes 2024-11-13T05:42:08,692 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:08,692 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:08,692 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. after waiting 0 ms 2024-11-13T05:42:08,692 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:08,692 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:08,693 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e31e188350bb5f6ecf6454cab120dda0: Waiting for close lock at 1731476528692Disabling compacts and flushes for region at 1731476528692Disabling writes for close at 1731476528692Writing region close event to WAL at 1731476528692Closed at 1731476528692 2024-11-13T05:42:08,695 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:42:08,696 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731476528696"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731476528696"}]},"ts":"1731476528696"} 2024-11-13T05:42:08,700 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:42:08,702 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:42:08,702 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476528702"}]},"ts":"1731476528702"} 2024-11-13T05:42:08,704 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T05:42:08,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, ASSIGN}] 2024-11-13T05:42:08,705 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, ASSIGN 2024-11-13T05:42:08,706 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45837,1731476380128; forceNewPlan=false, retain=false 2024-11-13T05:42:08,858 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=e31e188350bb5f6ecf6454cab120dda0, regionState=OPENING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:08,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, ASSIGN because future has completed 2024-11-13T05:42:08,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:42:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-13T05:42:09,020 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:09,020 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7752): Opening region: {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:42:09,020 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:09,020 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:42:09,020 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7794): checking encryption for e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:09,020 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7797): checking classloading for e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:09,022 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:09,024 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e31e188350bb5f6ecf6454cab120dda0 columnFamilyName A 2024-11-13T05:42:09,024 DEBUG [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:09,024 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] regionserver.HStore(327): Store=e31e188350bb5f6ecf6454cab120dda0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:42:09,024 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:09,026 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e31e188350bb5f6ecf6454cab120dda0 columnFamilyName B 2024-11-13T05:42:09,026 DEBUG [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:09,027 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] regionserver.HStore(327): Store=e31e188350bb5f6ecf6454cab120dda0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:42:09,027 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:09,028 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e31e188350bb5f6ecf6454cab120dda0 columnFamilyName C 2024-11-13T05:42:09,029 DEBUG [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:09,029 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] regionserver.HStore(327): Store=e31e188350bb5f6ecf6454cab120dda0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:42:09,029 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1038): replaying wal for e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:09,030 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:09,031 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:09,032 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1048): stopping wal replay for e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:09,032 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1060): Cleaning up temporary data for e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:09,033 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:42:09,035 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1093): writing seq id for e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:09,038 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:42:09,038 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1114): Opened e31e188350bb5f6ecf6454cab120dda0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62778931, jitterRate=-0.06452102959156036}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:42:09,038 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:09,039 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1006): Region open journal for e31e188350bb5f6ecf6454cab120dda0: Running coprocessor pre-open hook at 1731476529021Writing region info on filesystem at 1731476529021Initializing all the Stores at 1731476529021Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476529021Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476529022 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476529022Cleaning up temporary data from old regions at 1731476529033 (+11 ms)Running coprocessor post-open hooks at 1731476529038 (+5 ms)Region opened successfully at 1731476529039 (+1 ms) 2024-11-13T05:42:09,040 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., pid=130, masterSystemTime=1731476529014 2024-11-13T05:42:09,043 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:09,043 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:09,044 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=e31e188350bb5f6ecf6454cab120dda0, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:09,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:42:09,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-13T05:42:09,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; OpenRegionProcedure e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 in 185 msec 2024-11-13T05:42:09,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-13T05:42:09,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, ASSIGN in 346 msec 2024-11-13T05:42:09,055 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:42:09,055 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476529055"}]},"ts":"1731476529055"} 2024-11-13T05:42:09,057 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T05:42:09,058 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:42:09,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1970 sec 2024-11-13T05:42:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-13T05:42:10,017 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:10,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109f3221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:10,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:42:10,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:42:10,020 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:42:10,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:42:10,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:42:10,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34bf4c62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:10,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:42:10,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:42:10,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:10,022 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:42:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ed1c068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:10,024 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:42:10,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:42:10,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:42:10,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:42:10,028 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:42:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,37913,1731476379290 2024-11-13T05:42:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@daabe59 2024-11-13T05:42:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:42:10,030 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:42:10,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:42:10,032 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:42:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:10,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742355_1531 (size=995) 2024-11-13T05:42:10,485 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-13T05:42:10,486 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-13T05:42:10,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:42:10,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, REOPEN/MOVE}] 2024-11-13T05:42:10,497 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, REOPEN/MOVE 2024-11-13T05:42:10,498 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=e31e188350bb5f6ecf6454cab120dda0, regionState=CLOSING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:10,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, REOPEN/MOVE because future has completed 2024-11-13T05:42:10,499 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:42:10,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:42:10,651 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:10,651 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:42:10,652 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing e31e188350bb5f6ecf6454cab120dda0, disabling compactions & flushes 2024-11-13T05:42:10,652 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:10,652 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:10,652 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. after waiting 0 ms 2024-11-13T05:42:10,652 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:10,658 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:42:10,660 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:10,660 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for e31e188350bb5f6ecf6454cab120dda0: Waiting for close lock at 1731476530651Running coprocessor pre-close hooks at 1731476530651Disabling compacts and flushes for region at 1731476530651Disabling writes for close at 1731476530652 (+1 ms)Writing region close event to WAL at 1731476530652Running coprocessor post-close hooks at 1731476530659 (+7 ms)Closed at 1731476530660 (+1 ms) 2024-11-13T05:42:10,660 WARN [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionServer(3299): Not adding moved region record: e31e188350bb5f6ecf6454cab120dda0 to self. 2024-11-13T05:42:10,663 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:10,663 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=e31e188350bb5f6ecf6454cab120dda0, regionState=CLOSED 2024-11-13T05:42:10,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:42:10,667 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-13T05:42:10,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 in 167 msec 2024-11-13T05:42:10,668 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, REOPEN/MOVE; state=CLOSED, location=bc386b7bf2fe,45837,1731476380128; forceNewPlan=false, retain=true 2024-11-13T05:42:10,819 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=e31e188350bb5f6ecf6454cab120dda0, regionState=OPENING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:10,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, REOPEN/MOVE because future has completed 2024-11-13T05:42:10,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:42:10,983 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:10,984 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(7752): Opening region: {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:42:10,984 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:10,984 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:42:10,984 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(7794): checking encryption for e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:10,984 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(7797): checking classloading for e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:10,986 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:10,987 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e31e188350bb5f6ecf6454cab120dda0 columnFamilyName A 2024-11-13T05:42:10,989 DEBUG [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:10,990 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] regionserver.HStore(327): Store=e31e188350bb5f6ecf6454cab120dda0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:42:10,990 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:10,991 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e31e188350bb5f6ecf6454cab120dda0 columnFamilyName B 2024-11-13T05:42:10,991 DEBUG [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:10,992 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] regionserver.HStore(327): Store=e31e188350bb5f6ecf6454cab120dda0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:42:10,992 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:10,993 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e31e188350bb5f6ecf6454cab120dda0 columnFamilyName C 2024-11-13T05:42:10,993 DEBUG [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:10,994 INFO [StoreOpener-e31e188350bb5f6ecf6454cab120dda0-1 {}] regionserver.HStore(327): Store=e31e188350bb5f6ecf6454cab120dda0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:42:10,994 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1038): replaying wal for e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:10,995 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:10,996 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:10,997 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1048): stopping wal replay for e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:10,997 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1060): Cleaning up temporary data for e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:10,997 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:42:10,998 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1093): writing seq id for e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:10,999 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1114): Opened e31e188350bb5f6ecf6454cab120dda0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71432228, jitterRate=0.06442314386367798}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:42:10,999 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:11,000 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1006): Region open journal for e31e188350bb5f6ecf6454cab120dda0: Running coprocessor pre-open hook at 1731476530984Writing region info on filesystem at 1731476530984Initializing all the Stores at 1731476530985 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476530985Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476530986 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476530986Cleaning up temporary data from old regions at 1731476530997 (+11 ms)Running coprocessor post-open hooks at 1731476530999 (+2 ms)Region opened successfully at 1731476531000 (+1 ms) 2024-11-13T05:42:11,001 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., pid=135, masterSystemTime=1731476530977 2024-11-13T05:42:11,002 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:11,002 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:11,003 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=e31e188350bb5f6ecf6454cab120dda0, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:42:11,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=133 2024-11-13T05:42:11,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=133, state=SUCCESS, hasLock=false; OpenRegionProcedure e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 in 182 msec 2024-11-13T05:42:11,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-13T05:42:11,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, REOPEN/MOVE in 510 msec 2024-11-13T05:42:11,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-13T05:42:11,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-13T05:42:11,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 977 msec 2024-11-13T05:42:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T05:42:11,012 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:11,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:42:11,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:42:11,013 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:42:11,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:42:11,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:42:11,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f17ec53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:42:11,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:42:11,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:11,014 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:42:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3429d65f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22faf5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:42:11,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:42:11,016 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:42:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:42:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:42:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f512971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:42:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:42:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:11,017 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:42:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a43ca70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:42:11,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:42:11,018 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:42:11,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:42:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:42:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@687e59d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:42:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:42:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:11,019 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:42:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@530d5390, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,020 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:42:11,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:42:11,021 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:42:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:42:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:42:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ab0497, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:42:11,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:42:11,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:11,022 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:42:11,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79610a00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:42:11,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:42:11,024 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:42:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:42:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:42:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e527fe8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:42:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:42:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:11,025 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:42:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64104b33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,026 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:42:11,026 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:42:11,026 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:42:11,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:42:11,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:42:11,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@164570b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:42:11,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:42:11,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:11,027 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:42:11,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd5574d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:42:11,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:42:11,029 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:42:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:42:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:42:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@221eca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:42:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:42:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:11,030 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:42:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa7ebe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:42:11,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:42:11,031 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:42:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:42:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:42:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bb0724, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:42:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:42:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:11,033 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:42:11,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c91b35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:42:11,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:42:11,034 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:42:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:42:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:42:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b7418f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:42:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:42:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:11,035 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:42:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6248c92a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37913,-1 for getting cluster id 2024-11-13T05:42:11,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:42:11,037 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0872bdfa-3c79-4391-8d17-f5ae243c6af2' 2024-11-13T05:42:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:42:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0872bdfa-3c79-4391-8d17-f5ae243c6af2" 2024-11-13T05:42:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f7a8a8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37913,-1] 2024-11-13T05:42:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:42:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:11,038 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:42:11,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba7bd53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:42:11,039 DEBUG [Thread-2379 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:42:11,039 DEBUG [Thread-2387 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:42:11,039 DEBUG [Thread-2372 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:42:11,039 DEBUG [Thread-2376 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:42:11,039 DEBUG [Thread-2370 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:42:11,039 DEBUG [Thread-2368 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:42:11,039 DEBUG [Thread-2381 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:42:11,039 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:42:11,039 DEBUG [Thread-2383 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:42:11,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:42:11,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:42:11,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:42:11,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:42:11,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:42:11,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:42:11,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:42:11,040 DEBUG [Thread-2385 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:42:11,040 DEBUG [Thread-2374 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:42:11,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:42:11,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:42:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:42:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:42:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:42:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:42:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:42:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:42:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees 2024-11-13T05:42:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:42:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:42:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:42:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45837,1731476380128, seqNum=-1] 2024-11-13T05:42:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-13T05:42:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:42:11,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:42:11,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:42:11,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:42:11,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:42:11,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:42:11,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:42:11,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:42:11,043 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:42:11,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:42:11,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:42:11,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:42:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:42:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:42:11,044 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:42:11,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:42:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:42:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:42:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:42:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:42:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:42:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:42:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:42:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5] 2024-11-13T05:42:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:11,052 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:42:11,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T05:42:11,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137743c4a932a04632b4dfb23b0e02cba4_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476531050/Put/seqid=0 2024-11-13T05:42:11,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742356_1532 (size=14594) 2024-11-13T05:42:11,080 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:11,083 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137743c4a932a04632b4dfb23b0e02cba4_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137743c4a932a04632b4dfb23b0e02cba4_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:11,084 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/e2685fce33114621b9086b5f87fa0170, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:11,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476591078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/e2685fce33114621b9086b5f87fa0170 is 175, key is test_row_0/A:col10/1731476531050/Put/seqid=0 2024-11-13T05:42:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476591078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476591083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476591085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476591085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742357_1533 (size=39549) 2024-11-13T05:42:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-13T05:42:11,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T05:42:11,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:11,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:11,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:11,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476591197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476591197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476591197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476591197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476591197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T05:42:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:11,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-13T05:42:11,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476591406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476591406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476591406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476591406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476591409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,493 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/e2685fce33114621b9086b5f87fa0170 2024-11-13T05:42:11,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T05:42:11,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:11,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:11,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:11,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/63d0b216cfa441fb8f28bae4defc62be is 50, key is test_row_0/B:col10/1731476531050/Put/seqid=0 2024-11-13T05:42:11,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742358_1534 (size=12001) 2024-11-13T05:42:11,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T05:42:11,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:11,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:11,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:11,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-13T05:42:11,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476591717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476591717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476591717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476591717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476591717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:11,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T05:42:11,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:11,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:11,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:11,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:11,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/63d0b216cfa441fb8f28bae4defc62be 2024-11-13T05:42:11,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/c0034adf78b64c8699eaa3631e212eb8 is 50, key is test_row_0/C:col10/1731476531050/Put/seqid=0 2024-11-13T05:42:11,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742359_1535 (size=12001) 2024-11-13T05:42:11,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/c0034adf78b64c8699eaa3631e212eb8 2024-11-13T05:42:11,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/e2685fce33114621b9086b5f87fa0170 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/e2685fce33114621b9086b5f87fa0170 2024-11-13T05:42:11,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/e2685fce33114621b9086b5f87fa0170, entries=200, sequenceid=15, filesize=38.6 K 2024-11-13T05:42:11,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/63d0b216cfa441fb8f28bae4defc62be as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/63d0b216cfa441fb8f28bae4defc62be 2024-11-13T05:42:11,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/63d0b216cfa441fb8f28bae4defc62be, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T05:42:11,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/c0034adf78b64c8699eaa3631e212eb8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/c0034adf78b64c8699eaa3631e212eb8 2024-11-13T05:42:11,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/c0034adf78b64c8699eaa3631e212eb8, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T05:42:11,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e31e188350bb5f6ecf6454cab120dda0 in 905ms, sequenceid=15, compaction requested=false 2024-11-13T05:42:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T05:42:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:11,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-13T05:42:11,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:11,961 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T05:42:11,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138954d0d1b48f4b90995d94596f8a3535_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476531081/Put/seqid=0 2024-11-13T05:42:11,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742360_1536 (size=12154) 2024-11-13T05:42:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-13T05:42:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:12,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:12,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476592229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476592230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476592231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476592232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476592233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476592336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476592336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476592336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476592347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476592347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:12,376 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138954d0d1b48f4b90995d94596f8a3535_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138954d0d1b48f4b90995d94596f8a3535_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:12,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/55170f4b753b43d99fdac57f2822d34a, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:12,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/55170f4b753b43d99fdac57f2822d34a is 175, key is test_row_0/A:col10/1731476531081/Put/seqid=0 2024-11-13T05:42:12,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742361_1537 (size=30955) 2024-11-13T05:42:12,383 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/55170f4b753b43d99fdac57f2822d34a 2024-11-13T05:42:12,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/54f48c866eb14fed991cccbbe3c9897e is 50, key is test_row_0/B:col10/1731476531081/Put/seqid=0 2024-11-13T05:42:12,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742362_1538 (size=12001) 2024-11-13T05:42:12,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476592546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476592546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476592546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476592557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476592557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,636 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:42:12,793 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/54f48c866eb14fed991cccbbe3c9897e 2024-11-13T05:42:12,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/cbbc287791074f35a50a51a5550138e5 is 50, key is test_row_0/C:col10/1731476531081/Put/seqid=0 2024-11-13T05:42:12,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742363_1539 (size=12001) 2024-11-13T05:42:12,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476592856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476592856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476592856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476592866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476592866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-13T05:42:13,202 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/cbbc287791074f35a50a51a5550138e5 2024-11-13T05:42:13,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/55170f4b753b43d99fdac57f2822d34a as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/55170f4b753b43d99fdac57f2822d34a 2024-11-13T05:42:13,208 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/55170f4b753b43d99fdac57f2822d34a, entries=150, sequenceid=40, filesize=30.2 K 2024-11-13T05:42:13,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/54f48c866eb14fed991cccbbe3c9897e as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/54f48c866eb14fed991cccbbe3c9897e 2024-11-13T05:42:13,212 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/54f48c866eb14fed991cccbbe3c9897e, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T05:42:13,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/cbbc287791074f35a50a51a5550138e5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/cbbc287791074f35a50a51a5550138e5 2024-11-13T05:42:13,215 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/cbbc287791074f35a50a51a5550138e5, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T05:42:13,216 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e31e188350bb5f6ecf6454cab120dda0 in 1255ms, sequenceid=40, compaction requested=false 2024-11-13T05:42:13,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:13,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:13,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=137 2024-11-13T05:42:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=137 2024-11-13T05:42:13,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-13T05:42:13,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1730 sec 2024-11-13T05:42:13,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees in 2.1790 sec 2024-11-13T05:42:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:13,368 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:42:13,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T05:42:13,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139349f15060df4fb4a6dc5e9df85a4a96_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476533367/Put/seqid=0 2024-11-13T05:42:13,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742364_1540 (size=14594) 2024-11-13T05:42:13,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476593391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476593395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476593395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476593396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476593396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476593506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476593507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476593507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476593507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476593507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476593716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476593717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476593717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476593717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476593717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:13,779 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:13,782 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139349f15060df4fb4a6dc5e9df85a4a96_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139349f15060df4fb4a6dc5e9df85a4a96_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:13,782 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/b66486035c2d4cdbb70e13ab096b27da, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:13,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/b66486035c2d4cdbb70e13ab096b27da is 175, key is test_row_0/A:col10/1731476533367/Put/seqid=0 2024-11-13T05:42:13,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742365_1541 (size=39549) 2024-11-13T05:42:14,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476594026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476594026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476594026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:14,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476594027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476594027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:14,186 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/b66486035c2d4cdbb70e13ab096b27da 2024-11-13T05:42:14,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/50ccbbd166e94bfb85b0d90f67a792ea is 50, key is test_row_0/B:col10/1731476533367/Put/seqid=0 2024-11-13T05:42:14,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742366_1542 (size=12001) 2024-11-13T05:42:14,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476594536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:14,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476594536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:14,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476594536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476594536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:14,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476594537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:14,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/50ccbbd166e94bfb85b0d90f67a792ea 2024-11-13T05:42:14,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/8313ced49dfa4aab93206064a9172d90 is 50, key is test_row_0/C:col10/1731476533367/Put/seqid=0 2024-11-13T05:42:14,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742367_1543 (size=12001) 2024-11-13T05:42:15,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/8313ced49dfa4aab93206064a9172d90 2024-11-13T05:42:15,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/b66486035c2d4cdbb70e13ab096b27da as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/b66486035c2d4cdbb70e13ab096b27da 2024-11-13T05:42:15,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/b66486035c2d4cdbb70e13ab096b27da, entries=200, sequenceid=54, filesize=38.6 K 2024-11-13T05:42:15,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/50ccbbd166e94bfb85b0d90f67a792ea as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/50ccbbd166e94bfb85b0d90f67a792ea 2024-11-13T05:42:15,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/50ccbbd166e94bfb85b0d90f67a792ea, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T05:42:15,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/8313ced49dfa4aab93206064a9172d90 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/8313ced49dfa4aab93206064a9172d90 2024-11-13T05:42:15,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/8313ced49dfa4aab93206064a9172d90, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T05:42:15,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for e31e188350bb5f6ecf6454cab120dda0 in 1662ms, sequenceid=54, compaction requested=true 2024-11-13T05:42:15,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:42:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:15,031 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:42:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:15,031 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:42:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:15,031 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:15,031 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:15,031 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/B is initiating minor compaction (all files) 2024-11-13T05:42:15,031 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/A is initiating minor compaction (all files) 2024-11-13T05:42:15,032 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/A in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:15,032 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/B in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:15,032 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/63d0b216cfa441fb8f28bae4defc62be, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/54f48c866eb14fed991cccbbe3c9897e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/50ccbbd166e94bfb85b0d90f67a792ea] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=35.2 K 2024-11-13T05:42:15,032 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/e2685fce33114621b9086b5f87fa0170, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/55170f4b753b43d99fdac57f2822d34a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/b66486035c2d4cdbb70e13ab096b27da] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=107.5 K 2024-11-13T05:42:15,032 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:15,032 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/e2685fce33114621b9086b5f87fa0170, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/55170f4b753b43d99fdac57f2822d34a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/b66486035c2d4cdbb70e13ab096b27da] 2024-11-13T05:42:15,032 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 63d0b216cfa441fb8f28bae4defc62be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731476531050 2024-11-13T05:42:15,032 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2685fce33114621b9086b5f87fa0170, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731476531047 2024-11-13T05:42:15,032 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 54f48c866eb14fed991cccbbe3c9897e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731476531081 2024-11-13T05:42:15,032 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55170f4b753b43d99fdac57f2822d34a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731476531081 2024-11-13T05:42:15,032 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 50ccbbd166e94bfb85b0d90f67a792ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731476533367 2024-11-13T05:42:15,033 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting b66486035c2d4cdbb70e13ab096b27da, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731476532228 2024-11-13T05:42:15,037 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:15,038 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:15,038 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/bdcea2b69c0642468e6266fcc81e63c1 is 50, key is test_row_0/B:col10/1731476533367/Put/seqid=0 2024-11-13T05:42:15,039 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113069e7968655a4f7cbe8f1fd28424115d_e31e188350bb5f6ecf6454cab120dda0 store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:15,041 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113069e7968655a4f7cbe8f1fd28424115d_e31e188350bb5f6ecf6454cab120dda0, store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:15,042 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113069e7968655a4f7cbe8f1fd28424115d_e31e188350bb5f6ecf6454cab120dda0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:15,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742369_1545 (size=4469) 2024-11-13T05:42:15,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742368_1544 (size=12104) 2024-11-13T05:42:15,081 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/bdcea2b69c0642468e6266fcc81e63c1 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/bdcea2b69c0642468e6266fcc81e63c1 2024-11-13T05:42:15,085 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/B of e31e188350bb5f6ecf6454cab120dda0 into bdcea2b69c0642468e6266fcc81e63c1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:15,085 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:15,085 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/B, priority=13, startTime=1731476535031; duration=0sec 2024-11-13T05:42:15,085 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:15,085 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:B 2024-11-13T05:42:15,085 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:15,086 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:15,086 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/C is initiating minor compaction (all files) 2024-11-13T05:42:15,086 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/C in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:15,086 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/c0034adf78b64c8699eaa3631e212eb8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/cbbc287791074f35a50a51a5550138e5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/8313ced49dfa4aab93206064a9172d90] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=35.2 K 2024-11-13T05:42:15,087 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting c0034adf78b64c8699eaa3631e212eb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731476531050 2024-11-13T05:42:15,087 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting cbbc287791074f35a50a51a5550138e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731476531081 2024-11-13T05:42:15,087 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 8313ced49dfa4aab93206064a9172d90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731476533367 2024-11-13T05:42:15,092 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:15,093 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/ab9a473a9db64dad987468182c321875 is 50, key is test_row_0/C:col10/1731476533367/Put/seqid=0 2024-11-13T05:42:15,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742370_1546 (size=12104) 2024-11-13T05:42:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-13T05:42:15,207 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:15,207 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:42:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-11-13T05:42:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T05:42:15,209 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:42:15,209 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:42:15,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:42:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T05:42:15,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-13T05:42:15,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:15,362 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T05:42:15,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139d9948a4fcce4754992f66ed815ea20c_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476533390/Put/seqid=0 2024-11-13T05:42:15,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742371_1547 (size=12154) 2024-11-13T05:42:15,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:15,375 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139d9948a4fcce4754992f66ed815ea20c_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139d9948a4fcce4754992f66ed815ea20c_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:15,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/42834704d8b14074964b2b3c1af11afe, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:15,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/42834704d8b14074964b2b3c1af11afe is 175, key is test_row_0/A:col10/1731476533390/Put/seqid=0 2024-11-13T05:42:15,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742372_1548 (size=30955) 2024-11-13T05:42:15,478 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#A#compaction#460 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:15,479 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/89f39021088e4fe586a68ac4fdd2adae is 175, key is test_row_0/A:col10/1731476533367/Put/seqid=0 2024-11-13T05:42:15,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742373_1549 (size=31058) 2024-11-13T05:42:15,499 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/ab9a473a9db64dad987468182c321875 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/ab9a473a9db64dad987468182c321875 2024-11-13T05:42:15,507 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/C of e31e188350bb5f6ecf6454cab120dda0 into ab9a473a9db64dad987468182c321875(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:15,507 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:15,507 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/C, priority=13, startTime=1731476535031; duration=0sec 2024-11-13T05:42:15,507 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:15,507 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:C 2024-11-13T05:42:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T05:42:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:15,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:15,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476595583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476595583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:15,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476595583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:15,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476595584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476595585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:15,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476595697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476595697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:15,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476595697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476595697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476595697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:15,780 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/42834704d8b14074964b2b3c1af11afe 2024-11-13T05:42:15,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/42fc632662e846bfb3f90ceb0318935b is 50, key is test_row_0/B:col10/1731476533390/Put/seqid=0 2024-11-13T05:42:15,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742374_1550 (size=12001) 2024-11-13T05:42:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T05:42:15,886 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/89f39021088e4fe586a68ac4fdd2adae as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/89f39021088e4fe586a68ac4fdd2adae 2024-11-13T05:42:15,890 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/A of e31e188350bb5f6ecf6454cab120dda0 into 89f39021088e4fe586a68ac4fdd2adae(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:15,890 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:15,890 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/A, priority=13, startTime=1731476535031; duration=0sec 2024-11-13T05:42:15,890 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:15,890 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:A 2024-11-13T05:42:15,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476595906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476595906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476595906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:15,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476595906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476595906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,195 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/42fc632662e846bfb3f90ceb0318935b 2024-11-13T05:42:16,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/20080722b26b493c9b5cd88b86258cf0 is 50, key is test_row_0/C:col10/1731476533390/Put/seqid=0 2024-11-13T05:42:16,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742375_1551 (size=12001) 2024-11-13T05:42:16,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476596217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476596217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476596217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476596217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476596217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T05:42:16,604 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/20080722b26b493c9b5cd88b86258cf0 2024-11-13T05:42:16,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/42834704d8b14074964b2b3c1af11afe as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/42834704d8b14074964b2b3c1af11afe 2024-11-13T05:42:16,612 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/42834704d8b14074964b2b3c1af11afe, entries=150, sequenceid=78, filesize=30.2 K 2024-11-13T05:42:16,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/42fc632662e846bfb3f90ceb0318935b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/42fc632662e846bfb3f90ceb0318935b 2024-11-13T05:42:16,615 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/42fc632662e846bfb3f90ceb0318935b, entries=150, sequenceid=78, filesize=11.7 K 2024-11-13T05:42:16,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/20080722b26b493c9b5cd88b86258cf0 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/20080722b26b493c9b5cd88b86258cf0 2024-11-13T05:42:16,619 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/20080722b26b493c9b5cd88b86258cf0, entries=150, sequenceid=78, filesize=11.7 K 2024-11-13T05:42:16,619 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e31e188350bb5f6ecf6454cab120dda0 in 1258ms, sequenceid=78, compaction requested=false 2024-11-13T05:42:16,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:16,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:16,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-11-13T05:42:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=139 2024-11-13T05:42:16,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-13T05:42:16,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-13T05:42:16,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 1.4160 sec 2024-11-13T05:42:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:16,732 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:42:16,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T05:42:16,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136cb78ea3305f49a48c9ccc77759d3b0d_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476535584/Put/seqid=0 2024-11-13T05:42:16,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742376_1552 (size=12154) 2024-11-13T05:42:16,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476596752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476596753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476596754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476596755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476596755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476596866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476596866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476596867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476596867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476596867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476597076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476597077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476597077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476597077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476597077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,145 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:17,148 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136cb78ea3305f49a48c9ccc77759d3b0d_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136cb78ea3305f49a48c9ccc77759d3b0d_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:17,148 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/3687506b1540445fb6dff25dac557d4d, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:17,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/3687506b1540445fb6dff25dac557d4d is 175, key is test_row_0/A:col10/1731476535584/Put/seqid=0 2024-11-13T05:42:17,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742377_1553 (size=30955) 2024-11-13T05:42:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T05:42:17,356 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:17,356 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:42:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-13T05:42:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T05:42:17,358 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:42:17,358 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:42:17,358 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:42:17,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476597386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476597386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476597386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476597386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476597386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T05:42:17,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-13T05:42:17,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:17,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:17,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:17,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:17,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:17,552 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/3687506b1540445fb6dff25dac557d4d 2024-11-13T05:42:17,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/e2b86376b1d644bb878b3892f4d948ef is 50, key is test_row_0/B:col10/1731476535584/Put/seqid=0 2024-11-13T05:42:17,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742378_1554 (size=12001) 2024-11-13T05:42:17,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-13T05:42:17,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:17,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:17,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:17,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:17,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T05:42:17,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-13T05:42:17,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:17,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:17,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:17,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:17,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:17,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:17,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476597896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476597896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476597896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476597897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476597897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:17,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/e2b86376b1d644bb878b3892f4d948ef 2024-11-13T05:42:17,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/c1c6924bd57b4af0ad5f639648a4da7d is 50, key is test_row_0/C:col10/1731476535584/Put/seqid=0 2024-11-13T05:42:17,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-13T05:42:17,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:17,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:17,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:17,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:17,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:17,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742379_1555 (size=12001) 2024-11-13T05:42:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T05:42:18,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-13T05:42:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:18,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:18,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:18,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-13T05:42:18,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:18,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:18,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:18,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:18,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:18,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/c1c6924bd57b4af0ad5f639648a4da7d 2024-11-13T05:42:18,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/3687506b1540445fb6dff25dac557d4d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/3687506b1540445fb6dff25dac557d4d 2024-11-13T05:42:18,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/3687506b1540445fb6dff25dac557d4d, entries=150, sequenceid=95, filesize=30.2 K 2024-11-13T05:42:18,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/e2b86376b1d644bb878b3892f4d948ef as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/e2b86376b1d644bb878b3892f4d948ef 2024-11-13T05:42:18,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/e2b86376b1d644bb878b3892f4d948ef, entries=150, sequenceid=95, filesize=11.7 K 2024-11-13T05:42:18,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/c1c6924bd57b4af0ad5f639648a4da7d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/c1c6924bd57b4af0ad5f639648a4da7d 2024-11-13T05:42:18,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/c1c6924bd57b4af0ad5f639648a4da7d, entries=150, sequenceid=95, filesize=11.7 K 2024-11-13T05:42:18,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for e31e188350bb5f6ecf6454cab120dda0 in 1652ms, sequenceid=95, compaction requested=true 2024-11-13T05:42:18,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:18,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:42:18,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:18,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:42:18,384 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:18,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:18,384 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:18,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:42:18,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:18,385 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:18,385 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:18,385 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/A is initiating minor compaction (all files) 2024-11-13T05:42:18,385 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/B is initiating minor compaction (all files) 2024-11-13T05:42:18,385 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/B in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:18,385 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/A in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:18,385 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/89f39021088e4fe586a68ac4fdd2adae, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/42834704d8b14074964b2b3c1af11afe, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/3687506b1540445fb6dff25dac557d4d] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=90.8 K 2024-11-13T05:42:18,385 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/bdcea2b69c0642468e6266fcc81e63c1, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/42fc632662e846bfb3f90ceb0318935b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/e2b86376b1d644bb878b3892f4d948ef] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=35.3 K 2024-11-13T05:42:18,385 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:18,385 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/89f39021088e4fe586a68ac4fdd2adae, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/42834704d8b14074964b2b3c1af11afe, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/3687506b1540445fb6dff25dac557d4d] 2024-11-13T05:42:18,386 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89f39021088e4fe586a68ac4fdd2adae, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731476533367 2024-11-13T05:42:18,386 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting bdcea2b69c0642468e6266fcc81e63c1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731476533367 2024-11-13T05:42:18,386 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 42fc632662e846bfb3f90ceb0318935b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731476533390 2024-11-13T05:42:18,386 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42834704d8b14074964b2b3c1af11afe, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731476533390 2024-11-13T05:42:18,386 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting e2b86376b1d644bb878b3892f4d948ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731476535553 2024-11-13T05:42:18,386 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3687506b1540445fb6dff25dac557d4d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731476535553 2024-11-13T05:42:18,392 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:18,393 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113f248a46eb65643218c1bb4b2e4c1c89d_e31e188350bb5f6ecf6454cab120dda0 store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:18,393 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:18,394 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/146033bc093642d5bcb3de21442f2e26 is 50, key is test_row_0/B:col10/1731476535584/Put/seqid=0 2024-11-13T05:42:18,395 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113f248a46eb65643218c1bb4b2e4c1c89d_e31e188350bb5f6ecf6454cab120dda0, store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:18,395 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f248a46eb65643218c1bb4b2e4c1c89d_e31e188350bb5f6ecf6454cab120dda0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:18,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742380_1556 (size=12207) 2024-11-13T05:42:18,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742381_1557 (size=4469) 2024-11-13T05:42:18,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-13T05:42:18,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:18,425 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T05:42:18,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ecf9c8250bca40fe811b3234701326d1_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476536752/Put/seqid=0 2024-11-13T05:42:18,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742382_1558 (size=12154) 2024-11-13T05:42:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T05:42:18,805 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#A#compaction#469 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:18,806 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/a69cac1a61714786aac271892331d1f7 is 175, key is test_row_0/A:col10/1731476535584/Put/seqid=0 2024-11-13T05:42:18,808 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/146033bc093642d5bcb3de21442f2e26 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/146033bc093642d5bcb3de21442f2e26 2024-11-13T05:42:18,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742383_1559 (size=31161) 2024-11-13T05:42:18,812 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/B of e31e188350bb5f6ecf6454cab120dda0 into 146033bc093642d5bcb3de21442f2e26(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:18,812 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:18,812 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/B, priority=13, startTime=1731476538384; duration=0sec 2024-11-13T05:42:18,812 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:18,812 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:B 2024-11-13T05:42:18,812 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:18,813 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:18,813 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/C is initiating minor compaction (all files) 2024-11-13T05:42:18,813 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/C in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:18,813 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/ab9a473a9db64dad987468182c321875, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/20080722b26b493c9b5cd88b86258cf0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/c1c6924bd57b4af0ad5f639648a4da7d] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=35.3 K 2024-11-13T05:42:18,814 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting ab9a473a9db64dad987468182c321875, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731476533367 2024-11-13T05:42:18,814 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 20080722b26b493c9b5cd88b86258cf0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731476533390 2024-11-13T05:42:18,814 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting c1c6924bd57b4af0ad5f639648a4da7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731476535553 2024-11-13T05:42:18,820 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:18,820 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/2645ef9de7404c2ab21ad41df7d95a2d is 50, key is test_row_0/C:col10/1731476535584/Put/seqid=0 2024-11-13T05:42:18,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742384_1560 (size=12207) 2024-11-13T05:42:18,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:18,837 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ecf9c8250bca40fe811b3234701326d1_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ecf9c8250bca40fe811b3234701326d1_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:18,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/1780007da7384a4298ac7d4908efc250, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:18,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/1780007da7384a4298ac7d4908efc250 is 175, key is test_row_0/A:col10/1731476536752/Put/seqid=0 2024-11-13T05:42:18,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742385_1561 (size=30955) 2024-11-13T05:42:18,855 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/1780007da7384a4298ac7d4908efc250 2024-11-13T05:42:18,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/04e57491b0ff4624b44fccd767f0ba41 is 50, key is test_row_0/B:col10/1731476536752/Put/seqid=0 2024-11-13T05:42:18,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742386_1562 (size=12001) 2024-11-13T05:42:18,866 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/04e57491b0ff4624b44fccd767f0ba41 2024-11-13T05:42:18,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/102c69c2601e4fc5a019dba3b8515b50 is 50, key is test_row_0/C:col10/1731476536752/Put/seqid=0 2024-11-13T05:42:18,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742387_1563 (size=12001) 2024-11-13T05:42:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:18,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:18,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476598917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:18,946 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_0' in e31e188350bb5f6ecf6454cab120dda0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:42:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:18,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476598944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:18,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476598945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:18,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476598945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:18,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476598945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476599056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476599056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476599056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476599056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,214 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/a69cac1a61714786aac271892331d1f7 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/a69cac1a61714786aac271892331d1f7 2024-11-13T05:42:19,217 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/A of e31e188350bb5f6ecf6454cab120dda0 into a69cac1a61714786aac271892331d1f7(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:19,217 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:19,217 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/A, priority=13, startTime=1731476538384; duration=0sec 2024-11-13T05:42:19,218 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:19,218 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:A 2024-11-13T05:42:19,232 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/2645ef9de7404c2ab21ad41df7d95a2d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/2645ef9de7404c2ab21ad41df7d95a2d 2024-11-13T05:42:19,235 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/C of e31e188350bb5f6ecf6454cab120dda0 into 2645ef9de7404c2ab21ad41df7d95a2d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:19,235 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:19,235 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/C, priority=13, startTime=1731476538384; duration=0sec 2024-11-13T05:42:19,235 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:19,235 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:C 2024-11-13T05:42:19,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476599266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476599266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476599266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476599266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,275 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/102c69c2601e4fc5a019dba3b8515b50 2024-11-13T05:42:19,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/1780007da7384a4298ac7d4908efc250 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/1780007da7384a4298ac7d4908efc250 2024-11-13T05:42:19,283 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/1780007da7384a4298ac7d4908efc250, entries=150, sequenceid=117, filesize=30.2 K 2024-11-13T05:42:19,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/04e57491b0ff4624b44fccd767f0ba41 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/04e57491b0ff4624b44fccd767f0ba41 2024-11-13T05:42:19,286 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/04e57491b0ff4624b44fccd767f0ba41, entries=150, sequenceid=117, filesize=11.7 K 2024-11-13T05:42:19,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/102c69c2601e4fc5a019dba3b8515b50 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/102c69c2601e4fc5a019dba3b8515b50 2024-11-13T05:42:19,291 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/102c69c2601e4fc5a019dba3b8515b50, entries=150, sequenceid=117, filesize=11.7 K 2024-11-13T05:42:19,292 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e31e188350bb5f6ecf6454cab120dda0 in 867ms, sequenceid=117, compaction requested=false 2024-11-13T05:42:19,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:19,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:19,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-13T05:42:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-13T05:42:19,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-13T05:42:19,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9370 sec 2024-11-13T05:42:19,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 1.9410 sec 2024-11-13T05:42:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-13T05:42:19,506 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:19,507 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:42:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-13T05:42:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T05:42:19,508 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:42:19,509 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:42:19,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:42:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:19,580 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:42:19,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T05:42:19,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d432ceaa95e245ab9eb07240345b449e_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476539580/Put/seqid=0 2024-11-13T05:42:19,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742388_1564 (size=14794) 2024-11-13T05:42:19,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476599601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476599606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476599605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476599607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T05:42:19,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T05:42:19,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:19,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:19,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:19,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:19,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:19,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476599716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476599716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476599716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476599716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T05:42:19,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:19,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:19,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:19,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:19,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:19,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T05:42:19,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476599926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476599926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476599926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476599926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:19,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:19,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T05:42:19,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:19,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:19,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:19,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:19,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:19,991 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:19,995 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d432ceaa95e245ab9eb07240345b449e_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d432ceaa95e245ab9eb07240345b449e_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:19,995 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/357403fa47c142b4b7537e6d0d5eef7f, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:19,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/357403fa47c142b4b7537e6d0d5eef7f is 175, key is test_row_0/A:col10/1731476539580/Put/seqid=0 2024-11-13T05:42:19,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742389_1565 (size=39749) 2024-11-13T05:42:19,999 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/357403fa47c142b4b7537e6d0d5eef7f 2024-11-13T05:42:20,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/d4fcff039be94a3c9e5a8f9adc96501d is 50, key is test_row_0/B:col10/1731476539580/Put/seqid=0 2024-11-13T05:42:20,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742390_1566 (size=12151) 2024-11-13T05:42:20,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T05:42:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:20,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T05:42:20,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476600236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476600236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476600236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476600236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:20,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T05:42:20,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:20,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:20,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:20,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/d4fcff039be94a3c9e5a8f9adc96501d 2024-11-13T05:42:20,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/bb91ce3703d541e99c3ebd4ef4f3b0ed is 50, key is test_row_0/C:col10/1731476539580/Put/seqid=0 2024-11-13T05:42:20,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742391_1567 (size=12151) 2024-11-13T05:42:20,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T05:42:20,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:20,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:20,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:20,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T05:42:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:20,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T05:42:20,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T05:42:20,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:20,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:20,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:20,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:20,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476600746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476600746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:20,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476600746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:20,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:20,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476600746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:20,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/bb91ce3703d541e99c3ebd4ef4f3b0ed 2024-11-13T05:42:20,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/357403fa47c142b4b7537e6d0d5eef7f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/357403fa47c142b4b7537e6d0d5eef7f 2024-11-13T05:42:20,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/357403fa47c142b4b7537e6d0d5eef7f, entries=200, sequenceid=137, filesize=38.8 K 2024-11-13T05:42:20,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/d4fcff039be94a3c9e5a8f9adc96501d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/d4fcff039be94a3c9e5a8f9adc96501d 2024-11-13T05:42:20,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/d4fcff039be94a3c9e5a8f9adc96501d, entries=150, sequenceid=137, filesize=11.9 K 2024-11-13T05:42:20,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/bb91ce3703d541e99c3ebd4ef4f3b0ed as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/bb91ce3703d541e99c3ebd4ef4f3b0ed 2024-11-13T05:42:20,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/bb91ce3703d541e99c3ebd4ef4f3b0ed, entries=150, sequenceid=137, filesize=11.9 K 2024-11-13T05:42:20,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for e31e188350bb5f6ecf6454cab120dda0 in 1248ms, sequenceid=137, compaction requested=true 2024-11-13T05:42:20,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:20,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:42:20,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:20,829 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:20,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:42:20,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:20,829 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:20,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:42:20,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:20,829 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:20,829 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:20,830 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/A is initiating minor compaction (all files) 2024-11-13T05:42:20,830 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/B is initiating minor compaction (all files) 2024-11-13T05:42:20,830 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/A in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:20,830 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/B in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:20,830 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/a69cac1a61714786aac271892331d1f7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/1780007da7384a4298ac7d4908efc250, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/357403fa47c142b4b7537e6d0d5eef7f] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=99.5 K 2024-11-13T05:42:20,830 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/146033bc093642d5bcb3de21442f2e26, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/04e57491b0ff4624b44fccd767f0ba41, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/d4fcff039be94a3c9e5a8f9adc96501d] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=35.5 K 2024-11-13T05:42:20,830 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:20,830 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/a69cac1a61714786aac271892331d1f7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/1780007da7384a4298ac7d4908efc250, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/357403fa47c142b4b7537e6d0d5eef7f] 2024-11-13T05:42:20,830 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 146033bc093642d5bcb3de21442f2e26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731476535553 2024-11-13T05:42:20,830 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting a69cac1a61714786aac271892331d1f7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731476535553 2024-11-13T05:42:20,830 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1780007da7384a4298ac7d4908efc250, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731476536752 2024-11-13T05:42:20,830 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e57491b0ff4624b44fccd767f0ba41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731476536752 2024-11-13T05:42:20,830 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 357403fa47c142b4b7537e6d0d5eef7f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731476538918 2024-11-13T05:42:20,830 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting d4fcff039be94a3c9e5a8f9adc96501d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731476538918 2024-11-13T05:42:20,834 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:20,836 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113cd55b463ea594bf7a18a8bdfeb13fb17_e31e188350bb5f6ecf6454cab120dda0 store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:20,836 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#B#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:20,836 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/3d41227cdda44db29fedfb60233ce6b7 is 50, key is test_row_0/B:col10/1731476539580/Put/seqid=0 2024-11-13T05:42:20,838 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113cd55b463ea594bf7a18a8bdfeb13fb17_e31e188350bb5f6ecf6454cab120dda0, store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:20,838 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cd55b463ea594bf7a18a8bdfeb13fb17_e31e188350bb5f6ecf6454cab120dda0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:20,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742392_1568 (size=12459) 2024-11-13T05:42:20,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742393_1569 (size=4469) 2024-11-13T05:42:20,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-13T05:42:20,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:20,880 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T05:42:20,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ea51363b23934588a4ddaeb4ba9bdfbe_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476539605/Put/seqid=0 2024-11-13T05:42:20,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742394_1570 (size=12304) 2024-11-13T05:42:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:20,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:21,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476601013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:21,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476601126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:21,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:21,245 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/3d41227cdda44db29fedfb60233ce6b7 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/3d41227cdda44db29fedfb60233ce6b7 2024-11-13T05:42:21,245 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#A#compaction#478 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:21,246 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/3c4a0c79224a4ef9939a136159ddbcb8 is 175, key is test_row_0/A:col10/1731476539580/Put/seqid=0 2024-11-13T05:42:21,249 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/B of e31e188350bb5f6ecf6454cab120dda0 into 3d41227cdda44db29fedfb60233ce6b7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:21,249 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:21,249 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/B, priority=13, startTime=1731476540829; duration=0sec 2024-11-13T05:42:21,249 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:21,249 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:B 2024-11-13T05:42:21,249 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:21,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742395_1571 (size=31413) 2024-11-13T05:42:21,251 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:21,251 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/C is initiating minor compaction (all files) 2024-11-13T05:42:21,251 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/C in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:21,251 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/2645ef9de7404c2ab21ad41df7d95a2d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/102c69c2601e4fc5a019dba3b8515b50, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/bb91ce3703d541e99c3ebd4ef4f3b0ed] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=35.5 K 2024-11-13T05:42:21,251 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 2645ef9de7404c2ab21ad41df7d95a2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731476535553 2024-11-13T05:42:21,251 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 102c69c2601e4fc5a019dba3b8515b50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731476536752 2024-11-13T05:42:21,252 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting bb91ce3703d541e99c3ebd4ef4f3b0ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731476538918 2024-11-13T05:42:21,257 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#C#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:21,257 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/6402b9d4361146948a9e04ebc4b018f1 is 50, key is test_row_0/C:col10/1731476539580/Put/seqid=0 2024-11-13T05:42:21,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742396_1572 (size=12459) 2024-11-13T05:42:21,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:21,292 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ea51363b23934588a4ddaeb4ba9bdfbe_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ea51363b23934588a4ddaeb4ba9bdfbe_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:21,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/33e7f30fd0924a2c94aaf20615da1356, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:21,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/33e7f30fd0924a2c94aaf20615da1356 is 175, key is test_row_0/A:col10/1731476539605/Put/seqid=0 2024-11-13T05:42:21,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742397_1573 (size=31105) 2024-11-13T05:42:21,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476601337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476601646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:21,654 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/3c4a0c79224a4ef9939a136159ddbcb8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/3c4a0c79224a4ef9939a136159ddbcb8 2024-11-13T05:42:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T05:42:21,658 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/A of e31e188350bb5f6ecf6454cab120dda0 into 3c4a0c79224a4ef9939a136159ddbcb8(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:21,658 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:21,658 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/A, priority=13, startTime=1731476540829; duration=0sec 2024-11-13T05:42:21,658 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:21,658 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:A 2024-11-13T05:42:21,663 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/6402b9d4361146948a9e04ebc4b018f1 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/6402b9d4361146948a9e04ebc4b018f1 2024-11-13T05:42:21,666 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/C of e31e188350bb5f6ecf6454cab120dda0 into 6402b9d4361146948a9e04ebc4b018f1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:21,666 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:21,666 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/C, priority=13, startTime=1731476540829; duration=0sec 2024-11-13T05:42:21,666 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:21,666 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:C 2024-11-13T05:42:21,696 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/33e7f30fd0924a2c94aaf20615da1356 2024-11-13T05:42:21,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/50eb8027a2c443689317b46489a55fc0 is 50, key is test_row_0/B:col10/1731476539605/Put/seqid=0 2024-11-13T05:42:21,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742398_1574 (size=12151) 2024-11-13T05:42:21,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476601756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:21,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476601756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:21,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476601756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:21,757 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in e31e188350bb5f6ecf6454cab120dda0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:42:21,757 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_0' in e31e188350bb5f6ecf6454cab120dda0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:42:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:21,757 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_0' in e31e188350bb5f6ecf6454cab120dda0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:42:21,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:21,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476601766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:21,768 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_0' in e31e188350bb5f6ecf6454cab120dda0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:42:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:22,106 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/50eb8027a2c443689317b46489a55fc0 2024-11-13T05:42:22,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/3cec11e66b6c4d5a881f25ed2c12d5a5 is 50, key is test_row_0/C:col10/1731476539605/Put/seqid=0 2024-11-13T05:42:22,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742399_1575 (size=12151) 2024-11-13T05:42:22,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476602156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:22,516 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/3cec11e66b6c4d5a881f25ed2c12d5a5 2024-11-13T05:42:22,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/33e7f30fd0924a2c94aaf20615da1356 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/33e7f30fd0924a2c94aaf20615da1356 2024-11-13T05:42:22,522 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/33e7f30fd0924a2c94aaf20615da1356, entries=150, sequenceid=157, filesize=30.4 K 2024-11-13T05:42:22,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/50eb8027a2c443689317b46489a55fc0 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/50eb8027a2c443689317b46489a55fc0 2024-11-13T05:42:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,525 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/50eb8027a2c443689317b46489a55fc0, entries=150, sequenceid=157, filesize=11.9 K 2024-11-13T05:42:22,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/3cec11e66b6c4d5a881f25ed2c12d5a5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/3cec11e66b6c4d5a881f25ed2c12d5a5 2024-11-13T05:42:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,530 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/3cec11e66b6c4d5a881f25ed2c12d5a5, entries=150, sequenceid=157, filesize=11.9 K 2024-11-13T05:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,543 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e31e188350bb5f6ecf6454cab120dda0 in 1662ms, sequenceid=157, compaction requested=false 2024-11-13T05:42:22,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:22,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:22,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-13T05:42:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-13T05:42:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-13T05:42:22,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0350 sec 2024-11-13T05:42:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 3.0390 sec 2024-11-13T05:42:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,181 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:42:23,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T05:42:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130c4c34417f254086a85928573b5f0de6_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476541012/Put/seqid=0 2024-11-13T05:42:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742400_1576 (size=12304) 2024-11-13T05:42:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,191 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,194 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130c4c34417f254086a85928573b5f0de6_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130c4c34417f254086a85928573b5f0de6_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,195 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/f64aac505f8d437589307b239a4c7ce8, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/f64aac505f8d437589307b239a4c7ce8 is 175, key is test_row_0/A:col10/1731476541012/Put/seqid=0 2024-11-13T05:42:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742401_1577 (size=31101) 2024-11-13T05:42:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,199 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/f64aac505f8d437589307b239a4c7ce8 2024-11-13T05:42:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:23,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/edf5f50d2b70434da5a668ebd884baa5 is 50, key is test_row_0/B:col10/1731476541012/Put/seqid=0 2024-11-13T05:42:23,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742402_1578 (size=9757) 2024-11-13T05:42:23,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/edf5f50d2b70434da5a668ebd884baa5 2024-11-13T05:42:23,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/2536b866a1b24d868ccf6ac0c43be042 is 50, key is test_row_0/C:col10/1731476541012/Put/seqid=0 2024-11-13T05:42:23,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742403_1579 (size=9757) 2024-11-13T05:42:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476603264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:23,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476603376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:23,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476603587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:23,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:23,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/2536b866a1b24d868ccf6ac0c43be042 2024-11-13T05:42:23,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/f64aac505f8d437589307b239a4c7ce8 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/f64aac505f8d437589307b239a4c7ce8 2024-11-13T05:42:23,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/f64aac505f8d437589307b239a4c7ce8, entries=150, sequenceid=177, filesize=30.4 K 2024-11-13T05:42:23,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/edf5f50d2b70434da5a668ebd884baa5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/edf5f50d2b70434da5a668ebd884baa5 2024-11-13T05:42:23,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/edf5f50d2b70434da5a668ebd884baa5, entries=100, sequenceid=177, filesize=9.5 K 2024-11-13T05:42:23,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/2536b866a1b24d868ccf6ac0c43be042 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/2536b866a1b24d868ccf6ac0c43be042 2024-11-13T05:42:23,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/2536b866a1b24d868ccf6ac0c43be042, entries=100, sequenceid=177, filesize=9.5 K 2024-11-13T05:42:23,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for e31e188350bb5f6ecf6454cab120dda0 in 460ms, sequenceid=177, compaction requested=true 2024-11-13T05:42:23,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:23,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:42:23,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:23,641 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:23,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:42:23,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:23,641 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:23,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:42:23,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:23,642 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:23,642 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:23,642 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/B is initiating minor compaction (all files) 2024-11-13T05:42:23,642 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/A is initiating minor compaction (all files) 2024-11-13T05:42:23,642 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/A in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:23,642 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/B in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:23,642 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/3d41227cdda44db29fedfb60233ce6b7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/50eb8027a2c443689317b46489a55fc0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/edf5f50d2b70434da5a668ebd884baa5] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=33.6 K 2024-11-13T05:42:23,642 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/3c4a0c79224a4ef9939a136159ddbcb8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/33e7f30fd0924a2c94aaf20615da1356, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/f64aac505f8d437589307b239a4c7ce8] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=91.4 K 2024-11-13T05:42:23,642 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:23,642 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/3c4a0c79224a4ef9939a136159ddbcb8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/33e7f30fd0924a2c94aaf20615da1356, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/f64aac505f8d437589307b239a4c7ce8] 2024-11-13T05:42:23,642 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d41227cdda44db29fedfb60233ce6b7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731476538918 2024-11-13T05:42:23,642 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c4a0c79224a4ef9939a136159ddbcb8, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731476538918 2024-11-13T05:42:23,642 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 50eb8027a2c443689317b46489a55fc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731476539588 2024-11-13T05:42:23,642 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33e7f30fd0924a2c94aaf20615da1356, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731476539588 2024-11-13T05:42:23,642 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting edf5f50d2b70434da5a668ebd884baa5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731476541012 2024-11-13T05:42:23,642 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting f64aac505f8d437589307b239a4c7ce8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731476541008 2024-11-13T05:42:23,651 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:23,652 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:23,653 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113dc17e6aa299a4b29aaea8f73b8da2cfa_e31e188350bb5f6ecf6454cab120dda0 store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:23,653 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/d3577e359f2044718458b6d2475c8a43 is 50, key is test_row_0/B:col10/1731476541012/Put/seqid=0 2024-11-13T05:42:23,654 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113dc17e6aa299a4b29aaea8f73b8da2cfa_e31e188350bb5f6ecf6454cab120dda0, store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:23,654 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dc17e6aa299a4b29aaea8f73b8da2cfa_e31e188350bb5f6ecf6454cab120dda0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:23,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742404_1580 (size=4469) 2024-11-13T05:42:23,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742405_1581 (size=12561) 2024-11-13T05:42:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-13T05:42:23,676 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:23,677 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:42:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-13T05:42:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T05:42:23,678 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:42:23,679 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:42:23,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:42:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:23,779 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:42:23,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T05:42:23,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131c28dc458fb0465aadc0db0c434b8840_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476543778/Put/seqid=0 2024-11-13T05:42:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T05:42:23,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742406_1582 (size=12304) 2024-11-13T05:42:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476603825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:23,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-13T05:42:23,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:23,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:23,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:23,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:23,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476603830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476603830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476603831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:23,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:23,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476603896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:23,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476603936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:23,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:23,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476603946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:23,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476603946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:23,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476603946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:23,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-13T05:42:23,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:23,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:23,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:23,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:23,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T05:42:24,069 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:24,069 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/72d2fc5c510a42e6bc891e7c0353676f is 175, key is test_row_0/A:col10/1731476541012/Put/seqid=0 2024-11-13T05:42:24,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742407_1583 (size=31622) 2024-11-13T05:42:24,080 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/d3577e359f2044718458b6d2475c8a43 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/d3577e359f2044718458b6d2475c8a43 2024-11-13T05:42:24,083 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/B of e31e188350bb5f6ecf6454cab120dda0 into d3577e359f2044718458b6d2475c8a43(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:24,083 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:24,083 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/B, priority=13, startTime=1731476543641; duration=0sec 2024-11-13T05:42:24,083 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:24,083 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:B 2024-11-13T05:42:24,083 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:24,084 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:24,084 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/C is initiating minor compaction (all files) 2024-11-13T05:42:24,084 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/C in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:24,084 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/6402b9d4361146948a9e04ebc4b018f1, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/3cec11e66b6c4d5a881f25ed2c12d5a5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/2536b866a1b24d868ccf6ac0c43be042] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=33.6 K 2024-11-13T05:42:24,084 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 6402b9d4361146948a9e04ebc4b018f1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731476538918 2024-11-13T05:42:24,085 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cec11e66b6c4d5a881f25ed2c12d5a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731476539588 2024-11-13T05:42:24,085 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 2536b866a1b24d868ccf6ac0c43be042, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731476541012 2024-11-13T05:42:24,090 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#C#compaction#489 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:24,090 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/ac2900e8ec7e4e00be55df7d6aa27256 is 50, key is test_row_0/C:col10/1731476541012/Put/seqid=0 2024-11-13T05:42:24,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742408_1584 (size=12561) 2024-11-13T05:42:24,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-13T05:42:24,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:24,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:24,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:24,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476604146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:24,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476604156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476604156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476604157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:24,200 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:24,203 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131c28dc458fb0465aadc0db0c434b8840_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131c28dc458fb0465aadc0db0c434b8840_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:24,203 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/f99365103a084b5f8b611e764132d902, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:24,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/f99365103a084b5f8b611e764132d902 is 175, key is test_row_0/A:col10/1731476543778/Put/seqid=0 2024-11-13T05:42:24,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742409_1585 (size=31105) 2024-11-13T05:42:24,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-13T05:42:24,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:24,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:24,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:24,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T05:42:24,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476604407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:24,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-13T05:42:24,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:24,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:24,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:24,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476604456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:24,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476604466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:24,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476604466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:24,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476604466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:24,475 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/72d2fc5c510a42e6bc891e7c0353676f as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/72d2fc5c510a42e6bc891e7c0353676f 2024-11-13T05:42:24,479 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/A of e31e188350bb5f6ecf6454cab120dda0 into 72d2fc5c510a42e6bc891e7c0353676f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:24,479 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:24,479 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/A, priority=13, startTime=1731476543641; duration=0sec 2024-11-13T05:42:24,479 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:24,479 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:A 2024-11-13T05:42:24,497 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/ac2900e8ec7e4e00be55df7d6aa27256 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/ac2900e8ec7e4e00be55df7d6aa27256 2024-11-13T05:42:24,500 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/C of e31e188350bb5f6ecf6454cab120dda0 into ac2900e8ec7e4e00be55df7d6aa27256(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:24,500 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:24,500 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/C, priority=13, startTime=1731476543641; duration=0sec 2024-11-13T05:42:24,500 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:24,500 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:C 2024-11-13T05:42:24,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-13T05:42:24,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:24,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:24,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:24,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,607 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/f99365103a084b5f8b611e764132d902 2024-11-13T05:42:24,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/f7be5d7c70d84f30a30bf343a5e7adb5 is 50, key is test_row_0/B:col10/1731476543778/Put/seqid=0 2024-11-13T05:42:24,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742410_1586 (size=12151) 2024-11-13T05:42:24,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-13T05:42:24,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:24,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:24,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:24,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T05:42:24,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-13T05:42:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:24,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:24,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476604967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:24,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:24,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476604976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476604976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:24,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476604976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:25,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/f7be5d7c70d84f30a30bf343a5e7adb5 2024-11-13T05:42:25,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/121caf2923b545e1adc57be7aa1454a3 is 50, key is test_row_0/C:col10/1731476543778/Put/seqid=0 2024-11-13T05:42:25,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742411_1587 (size=12151) 2024-11-13T05:42:25,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-13T05:42:25,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:25,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:25,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:25,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:25,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:25,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-13T05:42:25,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:25,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:25,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:25,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:25,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:25,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-13T05:42:25,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:25,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:25,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:25,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:25,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:25,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476605416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:25,418 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in e31e188350bb5f6ecf6454cab120dda0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:42:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:25,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/121caf2923b545e1adc57be7aa1454a3 2024-11-13T05:42:25,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/f99365103a084b5f8b611e764132d902 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/f99365103a084b5f8b611e764132d902 2024-11-13T05:42:25,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/f99365103a084b5f8b611e764132d902, entries=150, sequenceid=198, filesize=30.4 K 2024-11-13T05:42:25,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/f7be5d7c70d84f30a30bf343a5e7adb5 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/f7be5d7c70d84f30a30bf343a5e7adb5 2024-11-13T05:42:25,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/f7be5d7c70d84f30a30bf343a5e7adb5, entries=150, sequenceid=198, filesize=11.9 K 2024-11-13T05:42:25,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/121caf2923b545e1adc57be7aa1454a3 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/121caf2923b545e1adc57be7aa1454a3 2024-11-13T05:42:25,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/121caf2923b545e1adc57be7aa1454a3, entries=150, sequenceid=198, filesize=11.9 K 2024-11-13T05:42:25,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for e31e188350bb5f6ecf6454cab120dda0 in 1658ms, sequenceid=198, compaction requested=false 2024-11-13T05:42:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:25,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-13T05:42:25,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:25,507 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T05:42:25,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138809f7847ffe42e09125fc77a8a2b838_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476543829/Put/seqid=0 2024-11-13T05:42:25,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742412_1588 (size=12304) 2024-11-13T05:42:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T05:42:25,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:25,921 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138809f7847ffe42e09125fc77a8a2b838_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138809f7847ffe42e09125fc77a8a2b838_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:25,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/8c33fbe8a1fe4b8ca78def12cf139808, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/8c33fbe8a1fe4b8ca78def12cf139808 is 175, key is test_row_0/A:col10/1731476543829/Put/seqid=0 2024-11-13T05:42:25,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742413_1589 (size=31105) 2024-11-13T05:42:25,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:25,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:26,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476606001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476606003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476606004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476606005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476606116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476606116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476606116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476606116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,325 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/8c33fbe8a1fe4b8ca78def12cf139808 2024-11-13T05:42:26,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476606327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476606327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476606327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476606327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/c1a91adbff904710ac6cc10bbde35897 is 50, key is test_row_0/B:col10/1731476543829/Put/seqid=0 2024-11-13T05:42:26,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742414_1590 (size=12151) 2024-11-13T05:42:26,335 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/c1a91adbff904710ac6cc10bbde35897 2024-11-13T05:42:26,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/196d64a416c94cfd981a738463ec96ca is 50, key is test_row_0/C:col10/1731476543829/Put/seqid=0 2024-11-13T05:42:26,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742415_1591 (size=12151) 2024-11-13T05:42:26,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476606636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476606636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476606636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476606637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:26,743 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/196d64a416c94cfd981a738463ec96ca 2024-11-13T05:42:26,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/8c33fbe8a1fe4b8ca78def12cf139808 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/8c33fbe8a1fe4b8ca78def12cf139808 2024-11-13T05:42:26,749 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/8c33fbe8a1fe4b8ca78def12cf139808, entries=150, sequenceid=217, filesize=30.4 K 2024-11-13T05:42:26,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/c1a91adbff904710ac6cc10bbde35897 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/c1a91adbff904710ac6cc10bbde35897 2024-11-13T05:42:26,753 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/c1a91adbff904710ac6cc10bbde35897, entries=150, sequenceid=217, filesize=11.9 K 2024-11-13T05:42:26,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/196d64a416c94cfd981a738463ec96ca as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/196d64a416c94cfd981a738463ec96ca 2024-11-13T05:42:26,756 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/196d64a416c94cfd981a738463ec96ca, entries=150, sequenceid=217, filesize=11.9 K 2024-11-13T05:42:26,756 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for e31e188350bb5f6ecf6454cab120dda0 in 1249ms, sequenceid=217, compaction requested=true 2024-11-13T05:42:26,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:26,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:26,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-13T05:42:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-13T05:42:26,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-13T05:42:26,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0790 sec 2024-11-13T05:42:26,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 3.0830 sec 2024-11-13T05:42:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:27,148 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:42:27,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T05:42:27,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ceb51e14ff364e9bb36086c3243eb000_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476546004/Put/seqid=0 2024-11-13T05:42:27,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742416_1592 (size=12304) 2024-11-13T05:42:27,158 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:27,160 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ceb51e14ff364e9bb36086c3243eb000_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ceb51e14ff364e9bb36086c3243eb000_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:27,161 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/7a53fdc8e651460592db49e103ddc289, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:27,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/7a53fdc8e651460592db49e103ddc289 is 175, key is test_row_0/A:col10/1731476546004/Put/seqid=0 2024-11-13T05:42:27,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742417_1593 (size=31105) 2024-11-13T05:42:27,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476607160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476607161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476607163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476607163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476607276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476607276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476607276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476607276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40386 deadline: 1731476607447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,450 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,45837,1731476380128 for 'test_row_2' in e31e188350bb5f6ecf6454cab120dda0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:42:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476607486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476607486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476607486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476607486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,565 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/7a53fdc8e651460592db49e103ddc289 2024-11-13T05:42:27,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/ac0aecf282114fddb38321440c62c5ef is 50, key is test_row_0/B:col10/1731476546004/Put/seqid=0 2024-11-13T05:42:27,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742418_1594 (size=12151) 2024-11-13T05:42:27,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/ac0aecf282114fddb38321440c62c5ef 2024-11-13T05:42:27,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/3d83a24e01a24ed3a1acde88e8376cbd is 50, key is test_row_0/C:col10/1731476546004/Put/seqid=0 2024-11-13T05:42:27,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742419_1595 (size=12151) 2024-11-13T05:42:27,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476607796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476607796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476607796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476607796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:27,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:27,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T05:42:27,837 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:27,837 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:42:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-13T05:42:27,839 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:42:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T05:42:27,839 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:42:27,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:42:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T05:42:27,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T05:42:27,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:27,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:27,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:27,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:27,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:28,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/3d83a24e01a24ed3a1acde88e8376cbd 2024-11-13T05:42:28,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/7a53fdc8e651460592db49e103ddc289 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/7a53fdc8e651460592db49e103ddc289 2024-11-13T05:42:28,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/7a53fdc8e651460592db49e103ddc289, entries=150, sequenceid=238, filesize=30.4 K 2024-11-13T05:42:28,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/ac0aecf282114fddb38321440c62c5ef as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/ac0aecf282114fddb38321440c62c5ef 2024-11-13T05:42:28,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/ac0aecf282114fddb38321440c62c5ef, entries=150, sequenceid=238, filesize=11.9 K 2024-11-13T05:42:28,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/3d83a24e01a24ed3a1acde88e8376cbd as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/3d83a24e01a24ed3a1acde88e8376cbd 2024-11-13T05:42:28,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/3d83a24e01a24ed3a1acde88e8376cbd, entries=150, sequenceid=238, filesize=11.9 K 2024-11-13T05:42:28,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for e31e188350bb5f6ecf6454cab120dda0 in 864ms, sequenceid=238, compaction requested=true 2024-11-13T05:42:28,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:28,012 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:42:28,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:42:28,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:28,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:42:28,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:28,013 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:42:28,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:42:28,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:28,013 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124937 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:42:28,013 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:42:28,013 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/A is initiating minor compaction (all files) 2024-11-13T05:42:28,013 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/B is initiating minor compaction (all files) 2024-11-13T05:42:28,013 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/B in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:28,013 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/A in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:28,013 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/d3577e359f2044718458b6d2475c8a43, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/f7be5d7c70d84f30a30bf343a5e7adb5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/c1a91adbff904710ac6cc10bbde35897, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/ac0aecf282114fddb38321440c62c5ef] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=47.9 K 2024-11-13T05:42:28,013 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/72d2fc5c510a42e6bc891e7c0353676f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/f99365103a084b5f8b611e764132d902, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/8c33fbe8a1fe4b8ca78def12cf139808, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/7a53fdc8e651460592db49e103ddc289] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=122.0 K 2024-11-13T05:42:28,013 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:28,013 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/72d2fc5c510a42e6bc891e7c0353676f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/f99365103a084b5f8b611e764132d902, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/8c33fbe8a1fe4b8ca78def12cf139808, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/7a53fdc8e651460592db49e103ddc289] 2024-11-13T05:42:28,014 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting d3577e359f2044718458b6d2475c8a43, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731476539588 2024-11-13T05:42:28,014 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72d2fc5c510a42e6bc891e7c0353676f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731476539588 2024-11-13T05:42:28,014 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting f7be5d7c70d84f30a30bf343a5e7adb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731476543262 2024-11-13T05:42:28,014 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting f99365103a084b5f8b611e764132d902, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731476543262 2024-11-13T05:42:28,014 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a91adbff904710ac6cc10bbde35897, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731476543785 2024-11-13T05:42:28,014 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c33fbe8a1fe4b8ca78def12cf139808, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731476543785 2024-11-13T05:42:28,014 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting ac0aecf282114fddb38321440c62c5ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731476546003 2024-11-13T05:42:28,014 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a53fdc8e651460592db49e103ddc289, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731476546003 2024-11-13T05:42:28,030 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:28,030 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#B#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:28,030 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/fedfa748b18949c58c069d52c7043dc9 is 50, key is test_row_0/B:col10/1731476546004/Put/seqid=0 2024-11-13T05:42:28,032 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111335b4d18d3a7c4e3695356b941733fffc_e31e188350bb5f6ecf6454cab120dda0 store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:28,035 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111335b4d18d3a7c4e3695356b941733fffc_e31e188350bb5f6ecf6454cab120dda0, store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:28,035 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111335b4d18d3a7c4e3695356b941733fffc_e31e188350bb5f6ecf6454cab120dda0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:28,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742420_1596 (size=12697) 2024-11-13T05:42:28,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742421_1597 (size=4469) 2024-11-13T05:42:28,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T05:42:28,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:28,145 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-13T05:42:28,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130425f1c0d8994a91806d5d35507f16ca_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476547162/Put/seqid=0 2024-11-13T05:42:28,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742422_1598 (size=12304) 2024-11-13T05:42:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T05:42:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:28,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:28,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476608330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476608331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476608333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476608336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476608446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476608446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476608446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476608446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,450 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/fedfa748b18949c58c069d52c7043dc9 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/fedfa748b18949c58c069d52c7043dc9 2024-11-13T05:42:28,454 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/B of e31e188350bb5f6ecf6454cab120dda0 into fedfa748b18949c58c069d52c7043dc9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:28,454 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:28,454 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/B, priority=12, startTime=1731476548013; duration=0sec 2024-11-13T05:42:28,454 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:28,454 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:B 2024-11-13T05:42:28,454 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:42:28,455 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:42:28,455 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/C is initiating minor compaction (all files) 2024-11-13T05:42:28,455 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/C in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:28,456 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/ac2900e8ec7e4e00be55df7d6aa27256, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/121caf2923b545e1adc57be7aa1454a3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/196d64a416c94cfd981a738463ec96ca, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/3d83a24e01a24ed3a1acde88e8376cbd] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=47.9 K 2024-11-13T05:42:28,456 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting ac2900e8ec7e4e00be55df7d6aa27256, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731476539588 2024-11-13T05:42:28,456 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 121caf2923b545e1adc57be7aa1454a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731476543262 2024-11-13T05:42:28,456 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 196d64a416c94cfd981a738463ec96ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731476543785 2024-11-13T05:42:28,457 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d83a24e01a24ed3a1acde88e8376cbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731476546003 2024-11-13T05:42:28,461 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#A#compaction#499 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:28,461 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/eb41d9f30d4d4a858c952864eb7b77d2 is 175, key is test_row_0/A:col10/1731476546004/Put/seqid=0 2024-11-13T05:42:28,463 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#C#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:28,464 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/d38cb7e1070c4241bc4022fadfe8dfd6 is 50, key is test_row_0/C:col10/1731476546004/Put/seqid=0 2024-11-13T05:42:28,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742423_1599 (size=31651) 2024-11-13T05:42:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T05:42:28,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742424_1600 (size=12697) 2024-11-13T05:42:28,484 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/d38cb7e1070c4241bc4022fadfe8dfd6 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/d38cb7e1070c4241bc4022fadfe8dfd6 2024-11-13T05:42:28,487 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/C of e31e188350bb5f6ecf6454cab120dda0 into d38cb7e1070c4241bc4022fadfe8dfd6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:28,487 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:28,487 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/C, priority=12, startTime=1731476548013; duration=0sec 2024-11-13T05:42:28,487 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:28,487 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:C 2024-11-13T05:42:28,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:28,559 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130425f1c0d8994a91806d5d35507f16ca_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130425f1c0d8994a91806d5d35507f16ca_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:28,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/7ab416b220154d399abe50494d8dd86d, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:28,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/7ab416b220154d399abe50494d8dd86d is 175, key is test_row_0/A:col10/1731476547162/Put/seqid=0 2024-11-13T05:42:28,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742425_1601 (size=31105) 2024-11-13T05:42:28,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476608656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476608656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476608656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476608656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,869 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/eb41d9f30d4d4a858c952864eb7b77d2 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/eb41d9f30d4d4a858c952864eb7b77d2 2024-11-13T05:42:28,873 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/A of e31e188350bb5f6ecf6454cab120dda0 into eb41d9f30d4d4a858c952864eb7b77d2(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:28,873 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:28,873 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/A, priority=12, startTime=1731476548012; duration=0sec 2024-11-13T05:42:28,873 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:28,873 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:A 2024-11-13T05:42:28,964 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/7ab416b220154d399abe50494d8dd86d 2024-11-13T05:42:28,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476608967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476608967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476608967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476608967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T05:42:28,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/a32c0591ae7547688704738cda22fb5b is 50, key is test_row_0/B:col10/1731476547162/Put/seqid=0 2024-11-13T05:42:28,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742426_1602 (size=12151) 2024-11-13T05:42:29,000 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/a32c0591ae7547688704738cda22fb5b 2024-11-13T05:42:29,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/12cfd6d401814d38ac93c5fa754cdea3 is 50, key is test_row_0/C:col10/1731476547162/Put/seqid=0 2024-11-13T05:42:29,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742427_1603 (size=12151) 2024-11-13T05:42:29,430 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/12cfd6d401814d38ac93c5fa754cdea3 2024-11-13T05:42:29,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/7ab416b220154d399abe50494d8dd86d as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/7ab416b220154d399abe50494d8dd86d 2024-11-13T05:42:29,436 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/7ab416b220154d399abe50494d8dd86d, entries=150, sequenceid=253, filesize=30.4 K 2024-11-13T05:42:29,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/a32c0591ae7547688704738cda22fb5b as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/a32c0591ae7547688704738cda22fb5b 2024-11-13T05:42:29,439 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/a32c0591ae7547688704738cda22fb5b, entries=150, sequenceid=253, filesize=11.9 K 2024-11-13T05:42:29,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/12cfd6d401814d38ac93c5fa754cdea3 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/12cfd6d401814d38ac93c5fa754cdea3 2024-11-13T05:42:29,443 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/12cfd6d401814d38ac93c5fa754cdea3, entries=150, sequenceid=253, filesize=11.9 K 2024-11-13T05:42:29,443 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for e31e188350bb5f6ecf6454cab120dda0 in 1298ms, sequenceid=253, compaction requested=false 2024-11-13T05:42:29,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:29,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:29,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-13T05:42:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-13T05:42:29,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-13T05:42:29,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6040 sec 2024-11-13T05:42:29,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.6120 sec 2024-11-13T05:42:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:29,478 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:42:29,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T05:42:29,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131e014c62248c444a80f21748a6aee4ee_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476549477/Put/seqid=0 2024-11-13T05:42:29,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742428_1604 (size=17534) 2024-11-13T05:42:29,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476609486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:29,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:29,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476609489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:29,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:29,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476609490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:29,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476609491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:29,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476609596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:29,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476609596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476609596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:29,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476609606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:29,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476609806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:29,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476609806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476609806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:29,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476609816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:29,888 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:29,892 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131e014c62248c444a80f21748a6aee4ee_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131e014c62248c444a80f21748a6aee4ee_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:29,892 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/6886fb55bf2f44189adb46bb262d5055, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:29,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/6886fb55bf2f44189adb46bb262d5055 is 175, key is test_row_0/A:col10/1731476549477/Put/seqid=0 2024-11-13T05:42:29,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742429_1605 (size=48639) 2024-11-13T05:42:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T05:42:29,997 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:29,997 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:42:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-13T05:42:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T05:42:29,999 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:42:30,000 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:42:30,000 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:42:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T05:42:30,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476610116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476610116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476610116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476610126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T05:42:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:30,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:30,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:30,297 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/6886fb55bf2f44189adb46bb262d5055 2024-11-13T05:42:30,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/7b52b96c138f4007823dc5b19331ab15 is 50, key is test_row_0/B:col10/1731476549477/Put/seqid=0 2024-11-13T05:42:30,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T05:42:30,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:30,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:30,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:30,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:30,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:42:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T05:42:30,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742430_1606 (size=12301) 2024-11-13T05:42:30,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/7b52b96c138f4007823dc5b19331ab15 2024-11-13T05:42:30,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/756b29b21e344adb81ffdc5ee7a765fb is 50, key is test_row_0/C:col10/1731476549477/Put/seqid=0 2024-11-13T05:42:30,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742431_1607 (size=12301) 2024-11-13T05:42:30,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/756b29b21e344adb81ffdc5ee7a765fb 2024-11-13T05:42:30,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/6886fb55bf2f44189adb46bb262d5055 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/6886fb55bf2f44189adb46bb262d5055 2024-11-13T05:42:30,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/6886fb55bf2f44189adb46bb262d5055, entries=250, sequenceid=279, filesize=47.5 K 2024-11-13T05:42:30,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/7b52b96c138f4007823dc5b19331ab15 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/7b52b96c138f4007823dc5b19331ab15 2024-11-13T05:42:30,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/7b52b96c138f4007823dc5b19331ab15, entries=150, sequenceid=279, filesize=12.0 K 2024-11-13T05:42:30,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/756b29b21e344adb81ffdc5ee7a765fb as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/756b29b21e344adb81ffdc5ee7a765fb 2024-11-13T05:42:30,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/756b29b21e344adb81ffdc5ee7a765fb, entries=150, sequenceid=279, filesize=12.0 K 2024-11-13T05:42:30,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e31e188350bb5f6ecf6454cab120dda0 in 919ms, sequenceid=279, compaction requested=true 2024-11-13T05:42:30,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:30,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:42:30,397 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:30,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:30,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:42:30,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:30,397 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:30,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:42:30,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:30,399 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:30,399 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111395 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:30,399 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/B is initiating minor compaction (all files) 2024-11-13T05:42:30,399 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/A is initiating minor compaction (all files) 2024-11-13T05:42:30,399 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/B in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:30,399 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/A in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:30,400 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/fedfa748b18949c58c069d52c7043dc9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/a32c0591ae7547688704738cda22fb5b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/7b52b96c138f4007823dc5b19331ab15] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=36.3 K 2024-11-13T05:42:30,400 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/eb41d9f30d4d4a858c952864eb7b77d2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/7ab416b220154d399abe50494d8dd86d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/6886fb55bf2f44189adb46bb262d5055] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=108.8 K 2024-11-13T05:42:30,400 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:30,400 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. files: [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/eb41d9f30d4d4a858c952864eb7b77d2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/7ab416b220154d399abe50494d8dd86d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/6886fb55bf2f44189adb46bb262d5055] 2024-11-13T05:42:30,401 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb41d9f30d4d4a858c952864eb7b77d2, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731476546003 2024-11-13T05:42:30,401 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting fedfa748b18949c58c069d52c7043dc9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731476546003 2024-11-13T05:42:30,401 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ab416b220154d399abe50494d8dd86d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731476547159 2024-11-13T05:42:30,401 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting a32c0591ae7547688704738cda22fb5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731476547159 2024-11-13T05:42:30,403 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b52b96c138f4007823dc5b19331ab15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731476548332 2024-11-13T05:42:30,403 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6886fb55bf2f44189adb46bb262d5055, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731476548329 2024-11-13T05:42:30,415 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#B#compaction#507 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:30,416 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/668484f547a3436eae5c7c2b6b5dc11a is 50, key is test_row_0/B:col10/1731476549477/Put/seqid=0 2024-11-13T05:42:30,417 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:30,423 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411134906761f18444bf985149a63bec090dd_e31e188350bb5f6ecf6454cab120dda0 store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:30,425 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411134906761f18444bf985149a63bec090dd_e31e188350bb5f6ecf6454cab120dda0, store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:30,426 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134906761f18444bf985149a63bec090dd_e31e188350bb5f6ecf6454cab120dda0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:30,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742432_1608 (size=12949) 2024-11-13T05:42:30,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742433_1609 (size=4469) 2024-11-13T05:42:30,462 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#A#compaction#508 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:30,462 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/1aaafd7d5767440fa7393fcafde10d00 is 175, key is test_row_0/A:col10/1731476549477/Put/seqid=0 2024-11-13T05:42:30,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T05:42:30,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:30,465 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T05:42:30,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742434_1610 (size=31903) 2024-11-13T05:42:30,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111368ba1092544148efa37afbf2236200bf_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476549489/Put/seqid=0 2024-11-13T05:42:30,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742435_1611 (size=12454) 2024-11-13T05:42:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T05:42:30,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. as already flushing 2024-11-13T05:42:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:30,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476610659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476610660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476610662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476610666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476610776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476610776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476610777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476610777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,849 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/668484f547a3436eae5c7c2b6b5dc11a as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/668484f547a3436eae5c7c2b6b5dc11a 2024-11-13T05:42:30,853 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/B of e31e188350bb5f6ecf6454cab120dda0 into 668484f547a3436eae5c7c2b6b5dc11a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:30,853 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:30,853 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/B, priority=13, startTime=1731476550397; duration=0sec 2024-11-13T05:42:30,853 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:30,853 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:B 2024-11-13T05:42:30,853 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:42:30,854 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:42:30,854 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1541): e31e188350bb5f6ecf6454cab120dda0/C is initiating minor compaction (all files) 2024-11-13T05:42:30,854 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e31e188350bb5f6ecf6454cab120dda0/C in TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:30,854 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/d38cb7e1070c4241bc4022fadfe8dfd6, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/12cfd6d401814d38ac93c5fa754cdea3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/756b29b21e344adb81ffdc5ee7a765fb] into tmpdir=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp, totalSize=36.3 K 2024-11-13T05:42:30,855 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting d38cb7e1070c4241bc4022fadfe8dfd6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731476546003 2024-11-13T05:42:30,855 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 12cfd6d401814d38ac93c5fa754cdea3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731476547159 2024-11-13T05:42:30,855 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] compactions.Compactor(225): Compacting 756b29b21e344adb81ffdc5ee7a765fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731476548332 2024-11-13T05:42:30,861 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e31e188350bb5f6ecf6454cab120dda0#C#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:42:30,862 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/85d9a7d582bb47c49ba286fdea317cd2 is 50, key is test_row_0/C:col10/1731476549477/Put/seqid=0 2024-11-13T05:42:30,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742436_1612 (size=12949) 2024-11-13T05:42:30,878 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/1aaafd7d5767440fa7393fcafde10d00 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/1aaafd7d5767440fa7393fcafde10d00 2024-11-13T05:42:30,881 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/A of e31e188350bb5f6ecf6454cab120dda0 into 1aaafd7d5767440fa7393fcafde10d00(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:30,882 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:30,882 INFO [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/A, priority=13, startTime=1731476550397; duration=0sec 2024-11-13T05:42:30,882 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:30,882 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:A 2024-11-13T05:42:30,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:30,887 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111368ba1092544148efa37afbf2236200bf_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111368ba1092544148efa37afbf2236200bf_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:30,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/81d7a489ad244d178380db20b0d8c440, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:30,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/81d7a489ad244d178380db20b0d8c440 is 175, key is test_row_0/A:col10/1731476549489/Put/seqid=0 2024-11-13T05:42:30,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742437_1613 (size=31255) 2024-11-13T05:42:30,891 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/81d7a489ad244d178380db20b0d8c440 2024-11-13T05:42:30,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/8c88571e6c304c0cbce59c5204f39858 is 50, key is test_row_0/B:col10/1731476549489/Put/seqid=0 2024-11-13T05:42:30,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742438_1614 (size=12301) 2024-11-13T05:42:30,901 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/8c88571e6c304c0cbce59c5204f39858 2024-11-13T05:42:30,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/278956b0e6174f2d859c15c94982d9e1 is 50, key is test_row_0/C:col10/1731476549489/Put/seqid=0 2024-11-13T05:42:30,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742439_1615 (size=12301) 2024-11-13T05:42:30,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476610986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476610986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476610986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476610986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:31,042 INFO [Thread-2383 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2383. 2024-11-13T05:42:31,042 DEBUG [Thread-2383 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:31,042 DEBUG [Thread-2383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,043 DEBUG [Thread-2383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:31,043 INFO [Thread-2381 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2381. 2024-11-13T05:42:31,043 DEBUG [Thread-2381 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:31,043 DEBUG [Thread-2381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,044 DEBUG [Thread-2381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:31,044 INFO [Thread-2385 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2385. 2024-11-13T05:42:31,044 DEBUG [Thread-2385 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:31,044 DEBUG [Thread-2385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,044 DEBUG [Thread-2385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:31,046 INFO [Thread-2387 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2387. 2024-11-13T05:42:31,046 DEBUG [Thread-2387 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:31,046 DEBUG [Thread-2387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,046 DEBUG [Thread-2387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:31,046 INFO [Thread-2379 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2379. 2024-11-13T05:42:31,046 DEBUG [Thread-2379 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:31,046 DEBUG [Thread-2379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,046 DEBUG [Thread-2379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T05:42:31,269 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/85d9a7d582bb47c49ba286fdea317cd2 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/85d9a7d582bb47c49ba286fdea317cd2 2024-11-13T05:42:31,273 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e31e188350bb5f6ecf6454cab120dda0/C of e31e188350bb5f6ecf6454cab120dda0 into 85d9a7d582bb47c49ba286fdea317cd2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:42:31,273 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:31,273 INFO [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., storeName=e31e188350bb5f6ecf6454cab120dda0/C, priority=13, startTime=1731476550397; duration=0sec 2024-11-13T05:42:31,273 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:31,273 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:C 2024-11-13T05:42:31,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:31,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:31,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40378 deadline: 1731476611296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40388 deadline: 1731476611297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40408 deadline: 1731476611296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:31,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:42:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1731476611297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 , the old value is region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:42:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., hostname=bc386b7bf2fe,45837,1731476380128, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:42:31,309 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/278956b0e6174f2d859c15c94982d9e1 2024-11-13T05:42:31,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/81d7a489ad244d178380db20b0d8c440 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/81d7a489ad244d178380db20b0d8c440 2024-11-13T05:42:31,315 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/81d7a489ad244d178380db20b0d8c440, entries=150, sequenceid=292, filesize=30.5 K 2024-11-13T05:42:31,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/8c88571e6c304c0cbce59c5204f39858 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/8c88571e6c304c0cbce59c5204f39858 2024-11-13T05:42:31,318 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/8c88571e6c304c0cbce59c5204f39858, entries=150, sequenceid=292, filesize=12.0 K 2024-11-13T05:42:31,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/278956b0e6174f2d859c15c94982d9e1 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/278956b0e6174f2d859c15c94982d9e1 2024-11-13T05:42:31,322 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/278956b0e6174f2d859c15c94982d9e1, entries=150, sequenceid=292, filesize=12.0 K 2024-11-13T05:42:31,323 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e31e188350bb5f6ecf6454cab120dda0 in 857ms, sequenceid=292, compaction requested=false 2024-11-13T05:42:31,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:31,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:31,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-13T05:42:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-13T05:42:31,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-13T05:42:31,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3240 sec 2024-11-13T05:42:31,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.3310 sec 2024-11-13T05:42:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45837 {}] regionserver.HRegion(8855): Flush requested on e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:31,468 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T05:42:31,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T05:42:31,468 INFO [Thread-2376 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2376. 2024-11-13T05:42:31,468 DEBUG [Thread-2376 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:31,468 DEBUG [Thread-2376 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,468 DEBUG [Thread-2376 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:31,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113609196196fce49fcb877ca8e4f14cdbc_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476550664/Put/seqid=0 2024-11-13T05:42:31,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742440_1616 (size=12454) 2024-11-13T05:42:31,808 INFO [Thread-2370 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2370. 2024-11-13T05:42:31,808 INFO [Thread-2372 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2372. 2024-11-13T05:42:31,808 DEBUG [Thread-2370 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:31,808 DEBUG [Thread-2372 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:31,808 DEBUG [Thread-2370 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,808 DEBUG [Thread-2372 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,808 INFO [Thread-2368 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2368. 2024-11-13T05:42:31,808 DEBUG [Thread-2368 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:31,808 DEBUG [Thread-2370 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,808 DEBUG [Thread-2372 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,808 DEBUG [Thread-2368 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,808 DEBUG [Thread-2368 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:31,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:31,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:31,810 INFO [Thread-2374 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2374. 2024-11-13T05:42:31,810 DEBUG [Thread-2374 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:42:31,810 DEBUG [Thread-2374 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,810 DEBUG [Thread-2374 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:31,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:31,879 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:31,882 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113609196196fce49fcb877ca8e4f14cdbc_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113609196196fce49fcb877ca8e4f14cdbc_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:31,883 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/18983850492b4fc49ebb075753f55069, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:31,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/18983850492b4fc49ebb075753f55069 is 175, key is test_row_0/A:col10/1731476550664/Put/seqid=0 2024-11-13T05:42:31,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742441_1617 (size=31255) 2024-11-13T05:42:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T05:42:32,147 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:32,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T05:42:32,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-13T05:42:32,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-13T05:42:32,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-13T05:42:32,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-13T05:42:32,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-13T05:42:32,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T05:42:32,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3898 2024-11-13T05:42:32,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3825 2024-11-13T05:42:32,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3753 2024-11-13T05:42:32,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3897 2024-11-13T05:42:32,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3829 2024-11-13T05:42:32,147 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T05:42:32,147 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:42:32,148 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:42:32,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:32,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:32,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:32,149 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T05:42:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T05:42:32,152 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476552152"}]},"ts":"1731476552152"} 2024-11-13T05:42:32,154 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T05:42:32,154 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T05:42:32,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:42:32,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, UNASSIGN}] 2024-11-13T05:42:32,158 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, UNASSIGN 2024-11-13T05:42:32,159 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=152 updating hbase:meta row=e31e188350bb5f6ecf6454cab120dda0, regionState=CLOSING, regionLocation=bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:32,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, UNASSIGN because future has completed 2024-11-13T05:42:32,161 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:42:32,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; CloseRegionProcedure e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128}] 2024-11-13T05:42:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T05:42:32,287 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/18983850492b4fc49ebb075753f55069 2024-11-13T05:42:32,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/70e1bdb1b002480c98a481d3fa572020 is 50, key is test_row_0/B:col10/1731476550664/Put/seqid=0 2024-11-13T05:42:32,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742442_1618 (size=12301) 2024-11-13T05:42:32,314 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(122): Close e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:32,314 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:42:32,314 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1722): Closing e31e188350bb5f6ecf6454cab120dda0, disabling compactions & flushes 2024-11-13T05:42:32,315 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T05:42:32,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/70e1bdb1b002480c98a481d3fa572020 2024-11-13T05:42:32,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/5640b7d6d1c44047ac308625272d581a is 50, key is test_row_0/C:col10/1731476550664/Put/seqid=0 2024-11-13T05:42:32,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742443_1619 (size=12301) 2024-11-13T05:42:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T05:42:33,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/5640b7d6d1c44047ac308625272d581a 2024-11-13T05:42:33,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/18983850492b4fc49ebb075753f55069 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/18983850492b4fc49ebb075753f55069 2024-11-13T05:42:33,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/18983850492b4fc49ebb075753f55069, entries=150, sequenceid=319, filesize=30.5 K 2024-11-13T05:42:33,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/70e1bdb1b002480c98a481d3fa572020 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/70e1bdb1b002480c98a481d3fa572020 2024-11-13T05:42:33,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/70e1bdb1b002480c98a481d3fa572020, entries=150, sequenceid=319, filesize=12.0 K 2024-11-13T05:42:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/5640b7d6d1c44047ac308625272d581a as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/5640b7d6d1c44047ac308625272d581a 2024-11-13T05:42:33,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/5640b7d6d1c44047ac308625272d581a, entries=150, sequenceid=319, filesize=12.0 K 2024-11-13T05:42:33,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=26.84 KB/27480 for e31e188350bb5f6ecf6454cab120dda0 in 1682ms, sequenceid=319, compaction requested=true 2024-11-13T05:42:33,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e31e188350bb5f6ecf6454cab120dda0: 2024-11-13T05:42:33,151 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:33,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:42:33,151 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:33,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:33,151 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. after waiting 0 ms 2024-11-13T05:42:33,151 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:33,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:42:33,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:42:33,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e31e188350bb5f6ecf6454cab120dda0:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:42:33,151 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(2902): Flushing e31e188350bb5f6ecf6454cab120dda0 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-13T05:42:33,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:42:33,151 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. because compaction request was cancelled 2024-11-13T05:42:33,151 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:A 2024-11-13T05:42:33,151 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. because compaction request was cancelled 2024-11-13T05:42:33,151 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. because compaction request was cancelled 2024-11-13T05:42:33,151 DEBUG [RS:0;bc386b7bf2fe:45837-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:B 2024-11-13T05:42:33,151 DEBUG [RS:0;bc386b7bf2fe:45837-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e31e188350bb5f6ecf6454cab120dda0:C 2024-11-13T05:42:33,155 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bfcfb1f37de34c43b16eeaa84cf5d79a_e31e188350bb5f6ecf6454cab120dda0 is 50, key is test_row_0/A:col10/1731476551806/Put/seqid=0 2024-11-13T05:42:33,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742444_1620 (size=9914) 2024-11-13T05:42:33,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T05:42:33,559 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:42:33,564 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bfcfb1f37de34c43b16eeaa84cf5d79a_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bfcfb1f37de34c43b16eeaa84cf5d79a_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:33,566 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/03f17c5ae403484f825047daf06a1512, store: [table=TestAcidGuarantees family=A region=e31e188350bb5f6ecf6454cab120dda0] 2024-11-13T05:42:33,567 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/03f17c5ae403484f825047daf06a1512 is 175, key is test_row_0/A:col10/1731476551806/Put/seqid=0 2024-11-13T05:42:33,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742445_1621 (size=22561) 2024-11-13T05:42:33,975 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/03f17c5ae403484f825047daf06a1512 2024-11-13T05:42:33,987 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/628f2d6bbe594a409d91829be4cd8ae7 is 50, key is test_row_0/B:col10/1731476551806/Put/seqid=0 2024-11-13T05:42:33,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742446_1622 (size=9857) 2024-11-13T05:42:34,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T05:42:34,392 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/628f2d6bbe594a409d91829be4cd8ae7 2024-11-13T05:42:34,406 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/f19449bd6f6147fc93214d1fd69ea658 is 50, key is test_row_0/C:col10/1731476551806/Put/seqid=0 2024-11-13T05:42:34,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742447_1623 (size=9857) 2024-11-13T05:42:34,811 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/f19449bd6f6147fc93214d1fd69ea658 2024-11-13T05:42:34,821 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/A/03f17c5ae403484f825047daf06a1512 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/03f17c5ae403484f825047daf06a1512 2024-11-13T05:42:34,827 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/03f17c5ae403484f825047daf06a1512, entries=100, sequenceid=326, filesize=22.0 K 2024-11-13T05:42:34,828 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/B/628f2d6bbe594a409d91829be4cd8ae7 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/628f2d6bbe594a409d91829be4cd8ae7 2024-11-13T05:42:34,832 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/628f2d6bbe594a409d91829be4cd8ae7, entries=100, sequenceid=326, filesize=9.6 K 2024-11-13T05:42:34,833 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/.tmp/C/f19449bd6f6147fc93214d1fd69ea658 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/f19449bd6f6147fc93214d1fd69ea658 2024-11-13T05:42:34,836 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/f19449bd6f6147fc93214d1fd69ea658, entries=100, sequenceid=326, filesize=9.6 K 2024-11-13T05:42:34,837 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for e31e188350bb5f6ecf6454cab120dda0 in 1686ms, sequenceid=326, compaction requested=true 2024-11-13T05:42:34,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/e2685fce33114621b9086b5f87fa0170, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/55170f4b753b43d99fdac57f2822d34a, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/b66486035c2d4cdbb70e13ab096b27da, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/89f39021088e4fe586a68ac4fdd2adae, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/42834704d8b14074964b2b3c1af11afe, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/a69cac1a61714786aac271892331d1f7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/3687506b1540445fb6dff25dac557d4d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/1780007da7384a4298ac7d4908efc250, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/357403fa47c142b4b7537e6d0d5eef7f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/3c4a0c79224a4ef9939a136159ddbcb8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/33e7f30fd0924a2c94aaf20615da1356, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/72d2fc5c510a42e6bc891e7c0353676f, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/f64aac505f8d437589307b239a4c7ce8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/f99365103a084b5f8b611e764132d902, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/8c33fbe8a1fe4b8ca78def12cf139808, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/eb41d9f30d4d4a858c952864eb7b77d2, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/7a53fdc8e651460592db49e103ddc289, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/7ab416b220154d399abe50494d8dd86d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/6886fb55bf2f44189adb46bb262d5055] to archive 2024-11-13T05:42:34,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:42:34,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/e2685fce33114621b9086b5f87fa0170 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/e2685fce33114621b9086b5f87fa0170 2024-11-13T05:42:34,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/55170f4b753b43d99fdac57f2822d34a to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/55170f4b753b43d99fdac57f2822d34a 2024-11-13T05:42:34,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/b66486035c2d4cdbb70e13ab096b27da to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/b66486035c2d4cdbb70e13ab096b27da 2024-11-13T05:42:34,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/89f39021088e4fe586a68ac4fdd2adae to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/89f39021088e4fe586a68ac4fdd2adae 2024-11-13T05:42:34,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/42834704d8b14074964b2b3c1af11afe to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/42834704d8b14074964b2b3c1af11afe 2024-11-13T05:42:34,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/a69cac1a61714786aac271892331d1f7 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/a69cac1a61714786aac271892331d1f7 2024-11-13T05:42:34,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/3687506b1540445fb6dff25dac557d4d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/3687506b1540445fb6dff25dac557d4d 2024-11-13T05:42:34,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/1780007da7384a4298ac7d4908efc250 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/1780007da7384a4298ac7d4908efc250 2024-11-13T05:42:34,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/357403fa47c142b4b7537e6d0d5eef7f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/357403fa47c142b4b7537e6d0d5eef7f 2024-11-13T05:42:34,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/3c4a0c79224a4ef9939a136159ddbcb8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/3c4a0c79224a4ef9939a136159ddbcb8 2024-11-13T05:42:34,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/33e7f30fd0924a2c94aaf20615da1356 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/33e7f30fd0924a2c94aaf20615da1356 2024-11-13T05:42:34,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/72d2fc5c510a42e6bc891e7c0353676f to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/72d2fc5c510a42e6bc891e7c0353676f 2024-11-13T05:42:34,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/f64aac505f8d437589307b239a4c7ce8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/f64aac505f8d437589307b239a4c7ce8 2024-11-13T05:42:34,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/f99365103a084b5f8b611e764132d902 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/f99365103a084b5f8b611e764132d902 2024-11-13T05:42:34,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/8c33fbe8a1fe4b8ca78def12cf139808 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/8c33fbe8a1fe4b8ca78def12cf139808 2024-11-13T05:42:34,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/eb41d9f30d4d4a858c952864eb7b77d2 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/eb41d9f30d4d4a858c952864eb7b77d2 2024-11-13T05:42:34,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/7a53fdc8e651460592db49e103ddc289 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/7a53fdc8e651460592db49e103ddc289 2024-11-13T05:42:34,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/7ab416b220154d399abe50494d8dd86d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/7ab416b220154d399abe50494d8dd86d 2024-11-13T05:42:34,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/6886fb55bf2f44189adb46bb262d5055 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/6886fb55bf2f44189adb46bb262d5055 2024-11-13T05:42:34,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/63d0b216cfa441fb8f28bae4defc62be, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/54f48c866eb14fed991cccbbe3c9897e, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/bdcea2b69c0642468e6266fcc81e63c1, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/50ccbbd166e94bfb85b0d90f67a792ea, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/42fc632662e846bfb3f90ceb0318935b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/146033bc093642d5bcb3de21442f2e26, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/e2b86376b1d644bb878b3892f4d948ef, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/04e57491b0ff4624b44fccd767f0ba41, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/3d41227cdda44db29fedfb60233ce6b7, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/d4fcff039be94a3c9e5a8f9adc96501d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/50eb8027a2c443689317b46489a55fc0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/d3577e359f2044718458b6d2475c8a43, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/edf5f50d2b70434da5a668ebd884baa5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/f7be5d7c70d84f30a30bf343a5e7adb5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/c1a91adbff904710ac6cc10bbde35897, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/fedfa748b18949c58c069d52c7043dc9, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/ac0aecf282114fddb38321440c62c5ef, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/a32c0591ae7547688704738cda22fb5b, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/7b52b96c138f4007823dc5b19331ab15] to archive 2024-11-13T05:42:34,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:42:34,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/63d0b216cfa441fb8f28bae4defc62be to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/63d0b216cfa441fb8f28bae4defc62be 2024-11-13T05:42:34,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/54f48c866eb14fed991cccbbe3c9897e to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/54f48c866eb14fed991cccbbe3c9897e 2024-11-13T05:42:34,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/bdcea2b69c0642468e6266fcc81e63c1 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/bdcea2b69c0642468e6266fcc81e63c1 2024-11-13T05:42:34,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/50ccbbd166e94bfb85b0d90f67a792ea to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/50ccbbd166e94bfb85b0d90f67a792ea 2024-11-13T05:42:34,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/42fc632662e846bfb3f90ceb0318935b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/42fc632662e846bfb3f90ceb0318935b 2024-11-13T05:42:34,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/146033bc093642d5bcb3de21442f2e26 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/146033bc093642d5bcb3de21442f2e26 2024-11-13T05:42:34,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/e2b86376b1d644bb878b3892f4d948ef to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/e2b86376b1d644bb878b3892f4d948ef 2024-11-13T05:42:34,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/04e57491b0ff4624b44fccd767f0ba41 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/04e57491b0ff4624b44fccd767f0ba41 2024-11-13T05:42:34,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/3d41227cdda44db29fedfb60233ce6b7 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/3d41227cdda44db29fedfb60233ce6b7 2024-11-13T05:42:34,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/d4fcff039be94a3c9e5a8f9adc96501d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/d4fcff039be94a3c9e5a8f9adc96501d 2024-11-13T05:42:34,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/50eb8027a2c443689317b46489a55fc0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/50eb8027a2c443689317b46489a55fc0 2024-11-13T05:42:34,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/d3577e359f2044718458b6d2475c8a43 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/d3577e359f2044718458b6d2475c8a43 2024-11-13T05:42:34,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/edf5f50d2b70434da5a668ebd884baa5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/edf5f50d2b70434da5a668ebd884baa5 2024-11-13T05:42:34,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/f7be5d7c70d84f30a30bf343a5e7adb5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/f7be5d7c70d84f30a30bf343a5e7adb5 2024-11-13T05:42:34,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/c1a91adbff904710ac6cc10bbde35897 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/c1a91adbff904710ac6cc10bbde35897 2024-11-13T05:42:34,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/fedfa748b18949c58c069d52c7043dc9 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/fedfa748b18949c58c069d52c7043dc9 2024-11-13T05:42:34,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/ac0aecf282114fddb38321440c62c5ef to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/ac0aecf282114fddb38321440c62c5ef 2024-11-13T05:42:34,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/a32c0591ae7547688704738cda22fb5b to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/a32c0591ae7547688704738cda22fb5b 2024-11-13T05:42:34,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/7b52b96c138f4007823dc5b19331ab15 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/7b52b96c138f4007823dc5b19331ab15 2024-11-13T05:42:34,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/c0034adf78b64c8699eaa3631e212eb8, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/cbbc287791074f35a50a51a5550138e5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/ab9a473a9db64dad987468182c321875, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/8313ced49dfa4aab93206064a9172d90, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/20080722b26b493c9b5cd88b86258cf0, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/2645ef9de7404c2ab21ad41df7d95a2d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/c1c6924bd57b4af0ad5f639648a4da7d, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/102c69c2601e4fc5a019dba3b8515b50, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/6402b9d4361146948a9e04ebc4b018f1, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/bb91ce3703d541e99c3ebd4ef4f3b0ed, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/3cec11e66b6c4d5a881f25ed2c12d5a5, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/ac2900e8ec7e4e00be55df7d6aa27256, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/2536b866a1b24d868ccf6ac0c43be042, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/121caf2923b545e1adc57be7aa1454a3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/196d64a416c94cfd981a738463ec96ca, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/d38cb7e1070c4241bc4022fadfe8dfd6, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/3d83a24e01a24ed3a1acde88e8376cbd, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/12cfd6d401814d38ac93c5fa754cdea3, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/756b29b21e344adb81ffdc5ee7a765fb] to archive 2024-11-13T05:42:34,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:42:34,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/c0034adf78b64c8699eaa3631e212eb8 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/c0034adf78b64c8699eaa3631e212eb8 2024-11-13T05:42:34,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/cbbc287791074f35a50a51a5550138e5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/cbbc287791074f35a50a51a5550138e5 2024-11-13T05:42:34,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/ab9a473a9db64dad987468182c321875 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/ab9a473a9db64dad987468182c321875 2024-11-13T05:42:34,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/8313ced49dfa4aab93206064a9172d90 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/8313ced49dfa4aab93206064a9172d90 2024-11-13T05:42:34,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/20080722b26b493c9b5cd88b86258cf0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/20080722b26b493c9b5cd88b86258cf0 2024-11-13T05:42:34,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/2645ef9de7404c2ab21ad41df7d95a2d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/2645ef9de7404c2ab21ad41df7d95a2d 2024-11-13T05:42:34,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/c1c6924bd57b4af0ad5f639648a4da7d to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/c1c6924bd57b4af0ad5f639648a4da7d 2024-11-13T05:42:34,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/102c69c2601e4fc5a019dba3b8515b50 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/102c69c2601e4fc5a019dba3b8515b50 2024-11-13T05:42:34,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/6402b9d4361146948a9e04ebc4b018f1 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/6402b9d4361146948a9e04ebc4b018f1 2024-11-13T05:42:34,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/bb91ce3703d541e99c3ebd4ef4f3b0ed to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/bb91ce3703d541e99c3ebd4ef4f3b0ed 2024-11-13T05:42:34,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/3cec11e66b6c4d5a881f25ed2c12d5a5 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/3cec11e66b6c4d5a881f25ed2c12d5a5 2024-11-13T05:42:34,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/ac2900e8ec7e4e00be55df7d6aa27256 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/ac2900e8ec7e4e00be55df7d6aa27256 2024-11-13T05:42:34,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/2536b866a1b24d868ccf6ac0c43be042 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/2536b866a1b24d868ccf6ac0c43be042 2024-11-13T05:42:34,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/121caf2923b545e1adc57be7aa1454a3 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/121caf2923b545e1adc57be7aa1454a3 2024-11-13T05:42:34,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/196d64a416c94cfd981a738463ec96ca to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/196d64a416c94cfd981a738463ec96ca 2024-11-13T05:42:34,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/d38cb7e1070c4241bc4022fadfe8dfd6 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/d38cb7e1070c4241bc4022fadfe8dfd6 2024-11-13T05:42:34,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/3d83a24e01a24ed3a1acde88e8376cbd to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/3d83a24e01a24ed3a1acde88e8376cbd 2024-11-13T05:42:34,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/12cfd6d401814d38ac93c5fa754cdea3 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/12cfd6d401814d38ac93c5fa754cdea3 2024-11-13T05:42:34,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/756b29b21e344adb81ffdc5ee7a765fb to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/756b29b21e344adb81ffdc5ee7a765fb 2024-11-13T05:42:34,914 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/recovered.edits/329.seqid, newMaxSeqId=329, maxSeqId=4 2024-11-13T05:42:34,914 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. 2024-11-13T05:42:34,914 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1676): Region close journal for e31e188350bb5f6ecf6454cab120dda0: Waiting for close lock at 1731476552314Running coprocessor pre-close hooks at 1731476552314Disabling compacts and flushes for region at 1731476552314Disabling writes for close at 1731476553151 (+837 ms)Obtaining lock to block concurrent updates at 1731476553151Preparing flush snapshotting stores in e31e188350bb5f6ecf6454cab120dda0 at 1731476553151Finished memstore snapshotting TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731476553151Flushing stores of TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0. at 1731476553151Flushing e31e188350bb5f6ecf6454cab120dda0/A: creating writer at 1731476553151Flushing mob file e31e188350bb5f6ecf6454cab120dda0/A: appending metadata at 1731476553155 (+4 ms)Flushing mob file e31e188350bb5f6ecf6454cab120dda0/A: closing flushed file at 1731476553155Flushing e31e188350bb5f6ecf6454cab120dda0/A: appending metadata at 1731476553566 (+411 ms)Flushing e31e188350bb5f6ecf6454cab120dda0/A: closing flushed file at 1731476553566Flushing e31e188350bb5f6ecf6454cab120dda0/B: creating writer at 1731476553985 (+419 ms)Flushing e31e188350bb5f6ecf6454cab120dda0/B: appending metadata at 1731476553987 (+2 ms)Flushing e31e188350bb5f6ecf6454cab120dda0/B: closing flushed file at 1731476553987Flushing e31e188350bb5f6ecf6454cab120dda0/C: creating writer at 1731476554402 (+415 ms)Flushing e31e188350bb5f6ecf6454cab120dda0/C: appending metadata at 1731476554405 (+3 ms)Flushing e31e188350bb5f6ecf6454cab120dda0/C: closing flushed file at 1731476554405Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d32088f: reopening flushed file at 1731476554820 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@595cbae5: reopening flushed file at 1731476554827 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64431cb5: reopening flushed file at 1731476554832 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for e31e188350bb5f6ecf6454cab120dda0 in 1686ms, sequenceid=326, compaction requested=true at 1731476554837 (+5 ms)Writing region close event to WAL at 1731476554911 (+74 ms)Running coprocessor post-close hooks at 1731476554914 (+3 ms)Closed at 1731476554914 2024-11-13T05:42:34,916 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(157): Closed e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:34,916 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=152 updating hbase:meta row=e31e188350bb5f6ecf6454cab120dda0, regionState=CLOSED 2024-11-13T05:42:34,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE, hasLock=false; CloseRegionProcedure e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 because future has completed 2024-11-13T05:42:34,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-13T05:42:34,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; CloseRegionProcedure e31e188350bb5f6ecf6454cab120dda0, server=bc386b7bf2fe,45837,1731476380128 in 2.7570 sec 2024-11-13T05:42:34,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-13T05:42:34,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e31e188350bb5f6ecf6454cab120dda0, UNASSIGN in 2.7630 sec 2024-11-13T05:42:34,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-13T05:42:34,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.7660 sec 2024-11-13T05:42:34,923 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476554923"}]},"ts":"1731476554923"} 2024-11-13T05:42:34,924 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T05:42:34,924 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T05:42:34,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.7750 sec 2024-11-13T05:42:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T05:42:36,308 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:36,309 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T05:42:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:36,312 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=154, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T05:42:36,314 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=154, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:36,317 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,320 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C, FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/recovered.edits] 2024-11-13T05:42:36,324 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/03f17c5ae403484f825047daf06a1512 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/03f17c5ae403484f825047daf06a1512 2024-11-13T05:42:36,326 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/18983850492b4fc49ebb075753f55069 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/18983850492b4fc49ebb075753f55069 2024-11-13T05:42:36,328 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/1aaafd7d5767440fa7393fcafde10d00 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/1aaafd7d5767440fa7393fcafde10d00 2024-11-13T05:42:36,330 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/81d7a489ad244d178380db20b0d8c440 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/A/81d7a489ad244d178380db20b0d8c440 2024-11-13T05:42:36,334 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/628f2d6bbe594a409d91829be4cd8ae7 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/628f2d6bbe594a409d91829be4cd8ae7 2024-11-13T05:42:36,336 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/668484f547a3436eae5c7c2b6b5dc11a to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/668484f547a3436eae5c7c2b6b5dc11a 2024-11-13T05:42:36,338 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/70e1bdb1b002480c98a481d3fa572020 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/70e1bdb1b002480c98a481d3fa572020 2024-11-13T05:42:36,340 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/8c88571e6c304c0cbce59c5204f39858 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/B/8c88571e6c304c0cbce59c5204f39858 2024-11-13T05:42:36,343 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/278956b0e6174f2d859c15c94982d9e1 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/278956b0e6174f2d859c15c94982d9e1 2024-11-13T05:42:36,344 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/5640b7d6d1c44047ac308625272d581a to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/5640b7d6d1c44047ac308625272d581a 2024-11-13T05:42:36,346 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/85d9a7d582bb47c49ba286fdea317cd2 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/85d9a7d582bb47c49ba286fdea317cd2 2024-11-13T05:42:36,347 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/f19449bd6f6147fc93214d1fd69ea658 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/C/f19449bd6f6147fc93214d1fd69ea658 2024-11-13T05:42:36,349 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/recovered.edits/329.seqid to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0/recovered.edits/329.seqid 2024-11-13T05:42:36,349 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/default/TestAcidGuarantees/e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,350 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T05:42:36,350 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T05:42:36,350 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T05:42:36,352 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130425f1c0d8994a91806d5d35507f16ca_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130425f1c0d8994a91806d5d35507f16ca_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,352 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130c4c34417f254086a85928573b5f0de6_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130c4c34417f254086a85928573b5f0de6_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,353 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131c28dc458fb0465aadc0db0c434b8840_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131c28dc458fb0465aadc0db0c434b8840_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,354 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131e014c62248c444a80f21748a6aee4ee_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131e014c62248c444a80f21748a6aee4ee_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,354 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113609196196fce49fcb877ca8e4f14cdbc_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113609196196fce49fcb877ca8e4f14cdbc_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,355 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111368ba1092544148efa37afbf2236200bf_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111368ba1092544148efa37afbf2236200bf_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,355 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136cb78ea3305f49a48c9ccc77759d3b0d_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136cb78ea3305f49a48c9ccc77759d3b0d_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,356 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137743c4a932a04632b4dfb23b0e02cba4_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137743c4a932a04632b4dfb23b0e02cba4_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,357 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138809f7847ffe42e09125fc77a8a2b838_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138809f7847ffe42e09125fc77a8a2b838_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,357 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138954d0d1b48f4b90995d94596f8a3535_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138954d0d1b48f4b90995d94596f8a3535_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,358 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139349f15060df4fb4a6dc5e9df85a4a96_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139349f15060df4fb4a6dc5e9df85a4a96_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,358 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139d9948a4fcce4754992f66ed815ea20c_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139d9948a4fcce4754992f66ed815ea20c_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,359 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bfcfb1f37de34c43b16eeaa84cf5d79a_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bfcfb1f37de34c43b16eeaa84cf5d79a_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,360 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ceb51e14ff364e9bb36086c3243eb000_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ceb51e14ff364e9bb36086c3243eb000_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,361 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d432ceaa95e245ab9eb07240345b449e_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d432ceaa95e245ab9eb07240345b449e_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,361 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ea51363b23934588a4ddaeb4ba9bdfbe_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ea51363b23934588a4ddaeb4ba9bdfbe_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,362 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ecf9c8250bca40fe811b3234701326d1_e31e188350bb5f6ecf6454cab120dda0 to hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ecf9c8250bca40fe811b3234701326d1_e31e188350bb5f6ecf6454cab120dda0 2024-11-13T05:42:36,362 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T05:42:36,364 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=154, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:36,367 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T05:42:36,368 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T05:42:36,369 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=154, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:36,369 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T05:42:36,369 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731476556369"}]},"ts":"9223372036854775807"} 2024-11-13T05:42:36,371 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T05:42:36,371 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e31e188350bb5f6ecf6454cab120dda0, NAME => 'TestAcidGuarantees,,1731476527862.e31e188350bb5f6ecf6454cab120dda0.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T05:42:36,371 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T05:42:36,371 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731476556371"}]},"ts":"9223372036854775807"} 2024-11-13T05:42:36,373 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T05:42:36,374 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=154, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:42:36,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 65 msec 2024-11-13T05:42:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37913 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T05:42:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T05:42:36,417 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:42:36,430 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 240) - Thread LEAK? -, OpenFileDescriptor=453 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 423) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7025 (was 7251) 2024-11-13T05:42:36,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T05:42:36,430 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:42:36,430 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:42:36,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:36,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:36,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:42:36,430 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T05:42:36,430 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=855483586, stopped=false 2024-11-13T05:42:36,431 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bc386b7bf2fe,37913,1731476379290 2024-11-13T05:42:36,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:42:36,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:42:36,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:42:36,496 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:42:36,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:42:36,498 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:42:36,498 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:42:36,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:36,498 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:42:36,499 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bc386b7bf2fe,45837,1731476380128' ***** 2024-11-13T05:42:36,499 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:42:36,500 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:42:36,500 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:42:36,501 INFO [RS:0;bc386b7bf2fe:45837 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:42:36,502 INFO [RS:0;bc386b7bf2fe:45837 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:42:36,502 INFO [regionserver/bc386b7bf2fe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:42:36,502 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(959): stopping server bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:36,502 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:42:36,502 INFO [RS:0;bc386b7bf2fe:45837 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bc386b7bf2fe:45837. 2024-11-13T05:42:36,503 DEBUG [RS:0;bc386b7bf2fe:45837 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:42:36,503 DEBUG [RS:0;bc386b7bf2fe:45837 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:36,503 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:42:36,504 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:42:36,504 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:42:36,504 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:42:36,504 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:42:36,504 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T05:42:36,504 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T05:42:36,505 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:42:36,505 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:42:36,505 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:42:36,505 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:42:36,505 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:42:36,505 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:42:36,505 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-13T05:42:36,525 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/.tmp/info/045f071fea724ed5920c30825d9ec5f1 is 95, key is TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe./info:/1731476416663/DeleteFamily/seqid=0 2024-11-13T05:42:36,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742448_1624 (size=6514) 2024-11-13T05:42:36,705 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:42:36,905 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:42:36,930 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/.tmp/info/045f071fea724ed5920c30825d9ec5f1 2024-11-13T05:42:36,953 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/.tmp/ns/01ee6cee75f24eaaa61d7ea6e43aae67 is 93, key is TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe./ns:/1731476416619/DeleteFamily/seqid=0 2024-11-13T05:42:36,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742449_1625 (size=5906) 2024-11-13T05:42:37,105 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:42:37,306 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:42:37,357 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/.tmp/ns/01ee6cee75f24eaaa61d7ea6e43aae67 2024-11-13T05:42:37,368 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T05:42:37,368 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T05:42:37,373 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/.tmp/rep_barrier/755e810438dc40e58865e61618bb8aa7 is 102, key is TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe./rep_barrier:/1731476416619/DeleteFamily/seqid=0 2024-11-13T05:42:37,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742450_1626 (size=6025) 2024-11-13T05:42:37,506 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:42:37,706 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:42:37,778 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/.tmp/rep_barrier/755e810438dc40e58865e61618bb8aa7 2024-11-13T05:42:37,805 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/.tmp/table/b8c9e0d8bdbb4756af1d63c8543557ce is 96, key is TestAcidGuarantees,,1731476384989.6abd824d4b7a94f2d19ab3e39e94ccfe./table:/1731476416619/DeleteFamily/seqid=0 2024-11-13T05:42:37,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742451_1627 (size=5936) 2024-11-13T05:42:37,907 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:42:38,107 DEBUG [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:42:38,208 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/.tmp/table/b8c9e0d8bdbb4756af1d63c8543557ce 2024-11-13T05:42:38,213 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/.tmp/info/045f071fea724ed5920c30825d9ec5f1 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/info/045f071fea724ed5920c30825d9ec5f1 2024-11-13T05:42:38,216 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/info/045f071fea724ed5920c30825d9ec5f1, entries=12, sequenceid=90, filesize=6.4 K 2024-11-13T05:42:38,217 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/.tmp/ns/01ee6cee75f24eaaa61d7ea6e43aae67 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/ns/01ee6cee75f24eaaa61d7ea6e43aae67 2024-11-13T05:42:38,221 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/ns/01ee6cee75f24eaaa61d7ea6e43aae67, entries=8, sequenceid=90, filesize=5.8 K 2024-11-13T05:42:38,222 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/.tmp/rep_barrier/755e810438dc40e58865e61618bb8aa7 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/rep_barrier/755e810438dc40e58865e61618bb8aa7 2024-11-13T05:42:38,226 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/rep_barrier/755e810438dc40e58865e61618bb8aa7, entries=6, sequenceid=90, filesize=5.9 K 2024-11-13T05:42:38,227 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/.tmp/table/b8c9e0d8bdbb4756af1d63c8543557ce as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/table/b8c9e0d8bdbb4756af1d63c8543557ce 2024-11-13T05:42:38,231 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/table/b8c9e0d8bdbb4756af1d63c8543557ce, entries=7, sequenceid=90, filesize=5.8 K 2024-11-13T05:42:38,232 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1727ms, sequenceid=90, compaction requested=false 2024-11-13T05:42:38,237 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-13T05:42:38,237 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:42:38,237 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:42:38,237 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476556504Running coprocessor pre-close hooks at 1731476556505 (+1 ms)Disabling compacts and flushes for region at 1731476556505Disabling writes for close at 1731476556505Obtaining lock to block concurrent updates at 1731476556505Preparing flush snapshotting stores in 1588230740 at 1731476556505Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731476556505Flushing stores of hbase:meta,,1.1588230740 at 1731476556506 (+1 ms)Flushing 1588230740/info: creating writer at 1731476556506Flushing 1588230740/info: appending metadata at 1731476556525 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731476556525Flushing 1588230740/ns: creating writer at 1731476556939 (+414 ms)Flushing 1588230740/ns: appending metadata at 1731476556953 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731476556953Flushing 1588230740/rep_barrier: creating writer at 1731476557360 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731476557373 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731476557373Flushing 1588230740/table: creating writer at 1731476557787 (+414 ms)Flushing 1588230740/table: appending metadata at 1731476557804 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731476557804Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ce2a912: reopening flushed file at 1731476558212 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9c8f23e: reopening flushed file at 1731476558217 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@530ae0ea: reopening flushed file at 1731476558221 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d71b9e0: reopening flushed file at 1731476558226 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1727ms, sequenceid=90, compaction requested=false at 1731476558232 (+6 ms)Writing region close event to WAL at 1731476558234 (+2 ms)Running coprocessor post-close hooks at 1731476558237 (+3 ms)Closed at 1731476558237 2024-11-13T05:42:38,238 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:42:38,307 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(976): stopping server bc386b7bf2fe,45837,1731476380128; all regions closed. 2024-11-13T05:42:38,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741834_1010 (size=25344) 2024-11-13T05:42:38,318 DEBUG [RS:0;bc386b7bf2fe:45837 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/oldWALs 2024-11-13T05:42:38,318 INFO [RS:0;bc386b7bf2fe:45837 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C45837%2C1731476380128.meta:.meta(num 1731476384548) 2024-11-13T05:42:38,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741832_1008 (size=13980477) 2024-11-13T05:42:38,322 DEBUG [RS:0;bc386b7bf2fe:45837 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/oldWALs 2024-11-13T05:42:38,322 INFO [RS:0;bc386b7bf2fe:45837 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C45837%2C1731476380128:(num 1731476383523) 2024-11-13T05:42:38,322 DEBUG [RS:0;bc386b7bf2fe:45837 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:42:38,322 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:42:38,322 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:42:38,322 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.ChoreService(370): Chore service for: regionserver/bc386b7bf2fe:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T05:42:38,322 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:42:38,323 INFO [regionserver/bc386b7bf2fe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:42:38,323 INFO [RS:0;bc386b7bf2fe:45837 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45837 2024-11-13T05:42:38,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bc386b7bf2fe,45837,1731476380128 2024-11-13T05:42:38,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:42:38,357 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:42:38,370 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bc386b7bf2fe,45837,1731476380128] 2024-11-13T05:42:38,382 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bc386b7bf2fe,45837,1731476380128 already deleted, retry=false 2024-11-13T05:42:38,382 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bc386b7bf2fe,45837,1731476380128 expired; onlineServers=0 2024-11-13T05:42:38,383 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bc386b7bf2fe,37913,1731476379290' ***** 2024-11-13T05:42:38,383 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T05:42:38,383 INFO [M:0;bc386b7bf2fe:37913 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:42:38,383 INFO [M:0;bc386b7bf2fe:37913 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:42:38,383 DEBUG [M:0;bc386b7bf2fe:37913 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T05:42:38,383 DEBUG [M:0;bc386b7bf2fe:37913 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T05:42:38,383 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T05:42:38,383 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731476383219 {}] cleaner.HFileCleaner(306): Exit Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731476383219,5,FailOnTimeoutGroup] 2024-11-13T05:42:38,383 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731476383219 {}] cleaner.HFileCleaner(306): Exit Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731476383219,5,FailOnTimeoutGroup] 2024-11-13T05:42:38,384 INFO [M:0;bc386b7bf2fe:37913 {}] hbase.ChoreService(370): Chore service for: master/bc386b7bf2fe:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T05:42:38,384 INFO [M:0;bc386b7bf2fe:37913 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:42:38,384 DEBUG [M:0;bc386b7bf2fe:37913 {}] master.HMaster(1795): Stopping service threads 2024-11-13T05:42:38,385 INFO [M:0;bc386b7bf2fe:37913 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T05:42:38,385 INFO [M:0;bc386b7bf2fe:37913 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:42:38,385 ERROR [M:0;bc386b7bf2fe:37913 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1228365231) connection to localhost/127.0.0.1:42759 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:42759,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-13T05:42:38,386 INFO [M:0;bc386b7bf2fe:37913 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:42:38,387 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T05:42:38,393 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:42:38,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T05:42:38,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:42:38,395 DEBUG [M:0;bc386b7bf2fe:37913 {}] zookeeper.ZKUtil(347): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T05:42:38,395 WARN [M:0;bc386b7bf2fe:37913 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T05:42:38,396 INFO [M:0;bc386b7bf2fe:37913 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/.lastflushedseqids 2024-11-13T05:42:38,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742452_1628 (size=72) 2024-11-13T05:42:38,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:42:38,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45837-0x10133fe66110001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:42:38,471 INFO [RS:0;bc386b7bf2fe:45837 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:42:38,472 INFO [RS:0;bc386b7bf2fe:45837 {}] regionserver.HRegionServer(1031): Exiting; stopping=bc386b7bf2fe,45837,1731476380128; zookeeper connection closed. 2024-11-13T05:42:38,472 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2c321248 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2c321248 2024-11-13T05:42:38,473 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T05:42:38,809 INFO [M:0;bc386b7bf2fe:37913 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:42:38,809 INFO [M:0;bc386b7bf2fe:37913 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T05:42:38,809 DEBUG [M:0;bc386b7bf2fe:37913 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:42:38,809 INFO [M:0;bc386b7bf2fe:37913 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:42:38,809 DEBUG [M:0;bc386b7bf2fe:37913 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:42:38,809 DEBUG [M:0;bc386b7bf2fe:37913 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:42:38,809 DEBUG [M:0;bc386b7bf2fe:37913 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:42:38,809 INFO [M:0;bc386b7bf2fe:37913 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=740.06 KB heapSize=908.86 KB 2024-11-13T05:42:38,834 DEBUG [M:0;bc386b7bf2fe:37913 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/42729e2f876b49c3ad58591a9b13aa5c is 82, key is hbase:meta,,1/info:regioninfo/1731476384629/Put/seqid=0 2024-11-13T05:42:38,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742453_1629 (size=5672) 2024-11-13T05:42:38,840 INFO [M:0;bc386b7bf2fe:37913 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2092 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/42729e2f876b49c3ad58591a9b13aa5c 2024-11-13T05:42:38,868 DEBUG [M:0;bc386b7bf2fe:37913 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/99a683f6a7f44e71ad0ebc30388f5938 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x83/proc:d/1731476531010/Put/seqid=0 2024-11-13T05:42:38,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742454_1630 (size=41342) 2024-11-13T05:42:39,272 INFO [M:0;bc386b7bf2fe:37913 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=739.50 KB at sequenceid=2092 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/99a683f6a7f44e71ad0ebc30388f5938 2024-11-13T05:42:39,280 INFO [M:0;bc386b7bf2fe:37913 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 99a683f6a7f44e71ad0ebc30388f5938 2024-11-13T05:42:39,297 DEBUG [M:0;bc386b7bf2fe:37913 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fad5de940d05408787e83daa9c3018a4 is 69, key is bc386b7bf2fe,45837,1731476380128/rs:state/1731476383268/Put/seqid=0 2024-11-13T05:42:39,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073742455_1631 (size=5156) 2024-11-13T05:42:39,701 INFO [M:0;bc386b7bf2fe:37913 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2092 (bloomFilter=true), to=hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fad5de940d05408787e83daa9c3018a4 2024-11-13T05:42:39,707 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:42:39,707 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T05:42:39,708 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T05:42:39,710 DEBUG [M:0;bc386b7bf2fe:37913 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/42729e2f876b49c3ad58591a9b13aa5c as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/42729e2f876b49c3ad58591a9b13aa5c 2024-11-13T05:42:39,714 INFO [M:0;bc386b7bf2fe:37913 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/42729e2f876b49c3ad58591a9b13aa5c, entries=8, sequenceid=2092, filesize=5.5 K 2024-11-13T05:42:39,715 DEBUG [M:0;bc386b7bf2fe:37913 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/99a683f6a7f44e71ad0ebc30388f5938 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/99a683f6a7f44e71ad0ebc30388f5938 2024-11-13T05:42:39,717 INFO [M:0;bc386b7bf2fe:37913 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 99a683f6a7f44e71ad0ebc30388f5938 2024-11-13T05:42:39,718 INFO [M:0;bc386b7bf2fe:37913 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/99a683f6a7f44e71ad0ebc30388f5938, entries=154, sequenceid=2092, filesize=40.4 K 2024-11-13T05:42:39,718 DEBUG [M:0;bc386b7bf2fe:37913 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fad5de940d05408787e83daa9c3018a4 as hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fad5de940d05408787e83daa9c3018a4 2024-11-13T05:42:39,721 INFO [M:0;bc386b7bf2fe:37913 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42759/user/jenkins/test-data/7cf89760-a5df-cf4a-96d1-c37465ae3745/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fad5de940d05408787e83daa9c3018a4, entries=1, sequenceid=2092, filesize=5.0 K 2024-11-13T05:42:39,721 INFO [M:0;bc386b7bf2fe:37913 {}] regionserver.HRegion(3140): Finished flush of dataSize ~740.06 KB/757820, heapSize ~908.56 KB/930368, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 912ms, sequenceid=2092, compaction requested=false 2024-11-13T05:42:39,722 INFO [M:0;bc386b7bf2fe:37913 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:42:39,723 DEBUG [M:0;bc386b7bf2fe:37913 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476558809Disabling compacts and flushes for region at 1731476558809Disabling writes for close at 1731476558809Obtaining lock to block concurrent updates at 1731476558809Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731476558809Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=757820, getHeapSize=930608, getOffHeapSize=0, getCellsCount=2271 at 1731476558810 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731476558810Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731476558810Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731476558833 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731476558833Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731476558844 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731476558867 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731476558867Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731476559280 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731476559296 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731476559296Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c6dca17: reopening flushed file at 1731476559708 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b5eed20: reopening flushed file at 1731476559714 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20ff0db: reopening flushed file at 1731476559718 (+4 ms)Finished flush of dataSize ~740.06 KB/757820, heapSize ~908.56 KB/930368, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 912ms, sequenceid=2092, compaction requested=false at 1731476559721 (+3 ms)Writing region close event to WAL at 1731476559722 (+1 ms)Closed at 1731476559722 2024-11-13T05:42:39,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41403 is added to blk_1073741830_1006 (size=894935) 2024-11-13T05:42:39,726 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:42:39,726 INFO [M:0;bc386b7bf2fe:37913 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T05:42:39,726 INFO [M:0;bc386b7bf2fe:37913 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37913 2024-11-13T05:42:39,726 INFO [M:0;bc386b7bf2fe:37913 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:42:39,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:42:39,883 INFO [M:0;bc386b7bf2fe:37913 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:42:39,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37913-0x10133fe66110000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:42:39,895 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:42:39,897 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:42:39,897 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:42:39,898 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:42:39,898 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/hadoop.log.dir/,STOPPED} 2024-11-13T05:42:39,901 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:42:39,901 WARN [BP-218274341-172.17.0.2-1731476375683 heartbeating to localhost/127.0.0.1:42759 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:42:39,901 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:42:39,901 WARN [BP-218274341-172.17.0.2-1731476375683 heartbeating to localhost/127.0.0.1:42759 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-218274341-172.17.0.2-1731476375683 (Datanode Uuid ba6fbef7-2527-4c14-8925-e4f52a867c14) service to localhost/127.0.0.1:42759 2024-11-13T05:42:39,904 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/cluster_5e8179f6-de47-de9a-0430-c54b93078463/data/data1/current/BP-218274341-172.17.0.2-1731476375683 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:42:39,904 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/cluster_5e8179f6-de47-de9a-0430-c54b93078463/data/data2/current/BP-218274341-172.17.0.2-1731476375683 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:42:39,905 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:42:39,911 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:42:39,912 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:42:39,912 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:42:39,912 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:42:39,912 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/26d2f1aa-fd15-154a-7f3e-2479bd4c3dab/hadoop.log.dir/,STOPPED} 2024-11-13T05:42:39,915 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T05:42:39,929 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T05:42:40,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down